DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

检查点数据

Browse Source
This commit is contained in:
ire 2025-02-08 17:10:19 +08:00
parent 58128ba243
commit e7d5551a0c
10 changed files with 84 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/vo/FineTuningTaskRespVO.java
View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.module.llm.controller.admin.finetuningtask.vo;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO;
import com.alibaba.fastjson.JSONObject;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -139,4 +140,6 @@ public class FineTuningTaskRespVO {
@Schema(description = "模型广场信息")
private AigcFineTuningDetailRespVO detailRespVO;
}
private JSONObject checkPointData;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelassesstaskmanual/vo/ModelAssessTaskManualSaveReqVO.java
View File

@ -40,4 +40,4 @@ public class ModelAssessTaskManualSaveReqVO {
@Schema(description ="模型类型（0普通，1官方）")
private Integer modelType;
}
}

8
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java
View File

@ -150,4 +150,12 @@ public class FineTuningTaskDO extends BaseDO {
private Long jobModelListId;
// 日志文件地址
private String trainLog;
//检查点文件地址
private String checkPointFilePath;
//检查点数据内容
private String checkPointData;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java
View File

@ -107,4 +107,6 @@ public class LLMBackendProperties {
* 知识库向量嵌入
*/
private String embed;
private String checkFileList;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java
View File

@ -17,6 +17,7 @@ import cn.iocoder.yudao.module.llm.service.async.AsyncFineTuningTaskService;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO;
import cn.iocoder.yudao.module.llm.service.modelservice.ModelServiceService;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
@ -176,6 +177,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
AigcFineTuningDetailRespVO aigcFineTuningDetailRespVO = trainHttpService.finetuningDetail(new HashMap<>(), fineTuningTaskDO.getJobId());
result.setDetailRespVO(aigcFineTuningDetailRespVO);
}
result.setCheckPointData(JSON.parseObject(fineTuningTaskDO.getCheckPointData()));
return result;
}

47
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java
View File

@ -15,6 +15,13 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;
@Component
@ -86,6 +93,44 @@ public class FineTuningTaskSyncService {
}catch (Exception e){
log.error(" error {}",e.getMessage());
}
try {
//获取检查点信息
String checkFileList = trainHttpService.getCheckFileList(jobModelName);
String checkpoint = "";
List<String> fileList = JSONArray.parseArray(checkFileList,String.class);
for (String s : fileList) {
if(s.contains("checkpoint")){
checkpoint = s;
}
}
String filePath = "/" + checkpoint + "/trainer_state.json";
String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath);
try {
URL url = new URL(fileUrl);
URLConnection urlConnection = url.openConnection();
InputStream inputStream = urlConnection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String line;
StringBuilder content = new StringBuilder();
while ((line = reader.readLine()) != null) {
content.append(line + "\n");
}
reader.close();
System.out.println(content.toString());
updateObj.setCheckPointData(content.toString());
updateObj.setCheckPointFilePath(fileUrl);
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}catch (Exception e){
log.error(" error {}",e.getMessage());
}
fineTuningTaskMapper.updateById(updateObj);
}
@ -98,4 +143,6 @@ public class FineTuningTaskSyncService {
}
}
}

11
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java
View File

@ -324,4 +324,15 @@ public class TrainHttpService {
String finetuningLog = llmBackendProperties.getFinetuningLog();
return HttpUtils.get(finetuningLog+params, null);
}
public String getCheckFileList(String name){
String checkFileList = llmBackendProperties.getCheckFileList();
return HttpUtils.get(checkFileList + name, null);
}
public String getCheckFile(String name,String path){
String checkFileList = llmBackendProperties.getCheckFileList();
return checkFileList + name + "&&file_path=" + path;
}
}

2
yudao-server/src/main/resources/application-dev.yaml
View File

@ -282,6 +282,8 @@ llm:
# 文生图
text_to_image: http://36.103.199.104:5123/generate-image
check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name=
# 知识库向量嵌入
embed: http://36.103.199.104:8123/embed
--- #################### iot相关配置 TODO 芋艿：再瞅瞅 ####################

6
yudao-server/src/main/resources/application-local.yaml
View File

@ -321,12 +321,14 @@ llm:
optimize_prompt: http://36.103.199.104:5123/optimize-prompt
auto_evaluation: http://36.103.199.104:5123/llm-eval
# 文生图
text_to_image: http://36.103.199.104:5123/generate-image
# 知识库向量嵌入
embed: http://36.103.199.104:8123/embed
check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name=
--- #################### iot相关配置 TODO 芋艿：再瞅瞅 ####################
iot:
emq:

5
yudao-server/src/main/resources/application-prod.yaml
View File

@ -321,12 +321,13 @@ llm:
optimize_prompt: http://36.133.1.230:5123/optimize-prompt
auto_evaluation: http://36.133.1.230:5123/llm-eval
# 文生图
text_to_image: http://36.133.1.230:5123/generate-image
# 知识库向量嵌入
embed: http://36.103.199.104:8123/embed
check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name=
--- #################### iot相关配置 TODO 芋艿：再瞅瞅 ####################
iot:
emq: