DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

检查点

Browse Source
This commit is contained in:
ire 2025-02-11 11:23:17 +08:00
parent 15f0c3c287
commit 973f925ca8
1 changed files with 28 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java
View File

@ -99,42 +99,47 @@ public class FineTuningTaskSyncService {
//todo 模型工厂的功能有问题，暂时写死
jobModelName = "Qwen2.5-0.5B-Instruct-147";
String checkFileList = trainHttpService.getCheckFileList(jobModelName);
String checkpoint = "";
List<String> checkpoints = new ArrayList<>();
List<String> fileUrls = new ArrayList<>();
List<String> fileList = JSONArray.parseArray(checkFileList,String.class);
Map<String,String> map = new HashMap<>();
for (String s : fileList) {
if(s.contains("checkpoint")){
checkpoint = s;
checkpoints.add(s);
}
}
String filePath = "/" + checkpoint + "/trainer_state.json";
String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath);
try {
URL url = new URL(fileUrl);
URLConnection urlConnection = url.openConnection();
InputStream inputStream = urlConnection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String line;
StringBuilder content = new StringBuilder();
while ((line = reader.readLine()) != null) {
content.append(line + "\n");
if(checkpoints.size() > 0){
for (String checkpoint : checkpoints) {
String filePath = "/" + checkpoint + "/trainer_state.json";
fileUrls.add(filePath);
String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath);
try {
URL url = new URL(fileUrl);
URLConnection urlConnection = url.openConnection();
InputStream inputStream = urlConnection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String line;
StringBuilder content = new StringBuilder();
while ((line = reader.readLine()) != null) {
content.append(line + "\n");
}
reader.close();
map.put(checkpoint,content.toString());
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
reader.close();
System.out.println(content.toString());
updateObj.setCheckPointData(content.toString());
updateObj.setCheckPointFilePath(fileUrl);
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
updateObj.setCheckPointFilePath(JSONObject.toJSONString(fileUrls));
updateObj.setCheckPointData(JSONObject.toJSONString(map));
}catch (Exception e){
log.error(" error {}",e.getMessage());
}
fineTuningTaskMapper.updateById(updateObj);
}
else {
updateObj.setStatus(status);