diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java index 64b652ff6..83f58d734 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java @@ -99,42 +99,47 @@ public class FineTuningTaskSyncService { //todo 模型工厂的功能有问题,暂时写死 jobModelName = "Qwen2.5-0.5B-Instruct-147"; String checkFileList = trainHttpService.getCheckFileList(jobModelName); - String checkpoint = ""; + List checkpoints = new ArrayList<>(); + List fileUrls = new ArrayList<>(); List fileList = JSONArray.parseArray(checkFileList,String.class); + Map map = new HashMap<>(); for (String s : fileList) { if(s.contains("checkpoint")){ - checkpoint = s; + checkpoints.add(s); } } - String filePath = "/" + checkpoint + "/trainer_state.json"; - String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath); - try { - URL url = new URL(fileUrl); - URLConnection urlConnection = url.openConnection(); - InputStream inputStream = urlConnection.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - StringBuilder content = new StringBuilder(); - while ((line = reader.readLine()) != null) { - content.append(line + "\n"); + if(checkpoints.size() > 0){ + for (String checkpoint : checkpoints) { + String filePath = "/" + checkpoint + "/trainer_state.json"; + fileUrls.add(filePath); + String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath); + try { + URL url = new URL(fileUrl); + URLConnection urlConnection = url.openConnection(); + InputStream inputStream = urlConnection.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + StringBuilder content = new StringBuilder(); + while ((line = reader.readLine()) != null) { + content.append(line + "\n"); + } + reader.close(); + map.put(checkpoint,content.toString()); + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } } - reader.close(); - System.out.println(content.toString()); - updateObj.setCheckPointData(content.toString()); - updateObj.setCheckPointFilePath(fileUrl); - } catch (MalformedURLException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); } + updateObj.setCheckPointFilePath(JSONObject.toJSONString(fileUrls)); + updateObj.setCheckPointData(JSONObject.toJSONString(map)); }catch (Exception e){ log.error(" error {}",e.getMessage()); } - fineTuningTaskMapper.updateById(updateObj); - } else { updateObj.setStatus(status);