DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

修复模型调优

Browse Source
This commit is contained in:
limin 2025-01-24 14:45:16 +08:00
parent 8b44edea3d
commit a4c6693872
13 changed files with 60 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/FineTuningTaskController.java
View File

@ -121,5 +121,13 @@ public class FineTuningTaskController {
ExcelUtils.write(response, "模型调优 —— 微调任务.xls", "数据", FineTuningTaskRespVO.class,
BeanUtils.toBean(list, FineTuningTaskRespVO.class));
}
@GetMapping("finetuning-log")
@Operation(summary = "获得模型微调日志")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
// @PreAuthorize("@ss.hasPermission('llm:fine-tuning:query')")
public CommonResult<String> getFineTuningLog(@RequestParam("id") Long id) {
String fineTuning = fineTuningTaskService.getFineTuningLog(id);
return success(fineTuning);
}
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java
View File

@ -148,4 +148,6 @@ public class FineTuningTaskDO extends BaseDO {
//AIGC模型列表主键id
private Long jobModelListId;
// 日志文件地址
private String trainLog;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java
View File

@ -71,6 +71,8 @@ public class LLMBackendProperties {
@NotNull(message = "创建微调任务 POST")
private String finetuningCreate;
private String finetuningLog;
private String finetuningDetail;
private String finetuningFileList;

17
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncFineTuningTaskService.java
View File

@ -1,13 +1,16 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetRespVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.finetuningtask.FineTuningTaskDO;
import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper;
import cn.iocoder.yudao.module.llm.enums.FinetuningTaskStatusEnum;
import cn.iocoder.yudao.module.llm.service.dataset.DatasetService;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningCreateReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningCreateRespVO;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Async;
@ -29,15 +32,22 @@ public class AsyncFineTuningTaskService {
@Lazy
@Resource
private DatasetService datasetService;
@Resource
private BaseModelMapper baseModelMapper;
//大模型平台创建调优任务
@Async
public void createTuning(FineTuningTaskDO fineTuningTask) {
try {
AigcFineTuningCreateReqVO req = getAigcFineTuningCreateReqVO(fineTuningTask);
BaseModelDO baseModelDO = baseModelMapper.selectById(fineTuningTask.getBaseModelId());
Long datasetId = fineTuningTask.getDataset();
DatasetRespVO dataset = datasetService.getDataset(datasetId);
// req.setFileId(dataset.getJobId());
if (baseModelDO != null){
req.setModel(baseModelDO.getAigcModelName());
}
req.setDataset(StringUtils.isNotBlank(dataset.getDatasetFileUrl())?dataset.getDatasetFileUrl():"storage/json/2025/01/_xVKpfDH8.json");
req.setDataset(dataset.getFileUrl());
req.setSuffix(String.valueOf(fineTuningTask.getId()));
AigcFineTuningCreateRespVO resp = trainHttpService.finetuningCreate(new HashMap<>(), req);
@ -47,6 +57,7 @@ public class AsyncFineTuningTaskService {
updateObj.setJobId(resp.getJobId());
updateObj.setStatus(FinetuningTaskStatusEnum.WAITING.getStatus());
updateObj.setJobModelName(resp.getFineTunedModel());
updateObj.setTrainLog(resp.getTrainLog());
} else {
updateObj.setStatus(FinetuningTaskStatusEnum.CANCELLED.getStatus());
}
@ -58,8 +69,8 @@ public class AsyncFineTuningTaskService {
private static AigcFineTuningCreateReqVO getAigcFineTuningCreateReqVO(FineTuningTaskDO fineTuningTask) {
AigcFineTuningCreateReqVO req = new AigcFineTuningCreateReqVO();
req.setBaseModel(fineTuningTask.getBaseModel());
req.setTrainEpoch(fineTuningTask.getEpoch());
req.setModel(fineTuningTask.getBaseModel());
req.setTrainEpoch( (fineTuningTask.getEpoch() == 0) ? 1 :fineTuningTask.getEpoch());
req.setSuffix(fineTuningTask.getTaskName());
req.setRemark(fineTuningTask.getTaskIntro());
req.setTrainBatchSize(fineTuningTask.getBatchSize());
@ -68,7 +79,7 @@ public class AsyncFineTuningTaskService {
req.setProcPerNode(fineTuningTask.getGpuCount());
req.setLearningRate(fineTuningTask.getLearningRate());
req.setModelMaxLength(fineTuningTask.getCutoffLen());
req.setLora(fineTuningTask.getLorayRank() != null);
// req.setLora(fineTuningTask.getLorayRank() != null);
return req;
}

1
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuning/FineTuningService.java
View File

@ -51,5 +51,4 @@ public interface FineTuningService {
* @return 模型微调分页
*/
PageResult<FineTuningDO> getFineTuningPage(FineTuningPageReqVO pageReqVO);
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskService.java
View File

@ -101,4 +101,6 @@ public interface FineTuningTaskService {
* @return 模型调优
*/
Map<Long, String> getModelTuningByDatasetId (Long datasetId);
String getFineTuningLog(Long id);
}

12
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java
View File

@ -19,6 +19,7 @@ import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import jodd.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -246,6 +247,17 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
return Collections.emptyMap();
}
@Override
public String getFineTuningLog(Long id) {
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(id);
if (fineTuningTaskDO != null && StringUtils.isNotBlank(fineTuningTaskDO.getTrainLog())){
Map<String,String> map = new HashMap<>();
map.put("log_path",fineTuningTaskDO.getTrainDuration());
return trainHttpService.getTrainLog(map);
}
return null;
}
@Override
public void stopFineTuningTask(Long id) {
FineTuningTaskDO fineTuningTaskDO = validateFineTuningTaskExists(id);

15
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java
View File

@ -11,6 +11,7 @@ import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
import kong.unirest.UnirestException;
import lombok.extern.slf4j.Slf4j;
import lombok.var;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@ -155,12 +156,9 @@ public class TrainHttpService {
String res = HttpUtils.post(llmBackendProperties.getFinetuningCreate(), headers, JSON.toJSONString(req));
log.info(" finetuningCreate:{}", res);
AigcRespVO aigcRespVO = JSON.parseObject(res, AigcRespVO.class);
if (aigcRespVO.isSuccess() && aigcRespVO.getData() != null){
AigcFineTuningCreateRespVO aigcFineTuningCreateRespVO = JSON.parseObject(aigcRespVO.getData().toString(), AigcFineTuningCreateRespVO.class);
log.info(" finetuningCreate:{}", aigcFineTuningCreateRespVO);
return aigcFineTuningCreateRespVO;
}
return null;
var aigcFineTuningCreateRespVO = JSON.parseObject(res, AigcFineTuningCreateRespVO.class);
log.info(" finetuningCreate:{}", aigcFineTuningCreateRespVO);
return aigcFineTuningCreateRespVO;
}
/**
* 根据表名称查询数据
@ -321,4 +319,9 @@ public class TrainHttpService {
return aigcRespVO;
}
public String getTrainLog(Map<String, String> params) {
String finetuningLog = llmBackendProperties.getFinetuningLog();
return HttpUtils.get(finetuningLog, params);
}
}

6
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningCreateReqVO.java
View File

@ -10,9 +10,9 @@ import lombok.NoArgsConstructor;
public class AigcFineTuningCreateReqVO {
private String scenario = "general";
private String fileId = "6237ed4d-a046-479c-80d6-8579a0283994";
// private String fileId = "6237ed4d-a046-479c-80d6-8579a0283994";
private String dataset;
private String baseModel;
private String model;
private int trainEpoch;
private String suffix;
private String remark;
@ -22,5 +22,5 @@ public class AigcFineTuningCreateReqVO {
private int procPerNode;
private double learningRate;
private int modelMaxLength;
private boolean lora;
// private boolean lora;
}

4
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningCreateRespVO.java
View File

@ -4,7 +4,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.Map;
@ -22,7 +22,7 @@ public class AigcFineTuningCreateRespVO {
private String fineTunedModel;
private String remark;
private String finishedAt;
private ZonedDateTime createdAt;
private String createdAt;
private String trainPublisher;
private String trainLog;
private String errorMessage;

2
yudao-server/src/main/resources/application-dev.yaml
View File

@ -243,6 +243,8 @@ llm:
finetuning_create: http://36.103.199.104:5123/llm/finetuning
# 微调任务详情 GET
finetuning_detail: http://36.103.199.104:9000/api/finetuning
# 日志获取
finetuning_log: http://36.103.199.104:5123/llm/get_log
# 微调任务取消
finetuning_cancel: http://36.103.199.104:9000/api/finetuning/%s/cancel
# 微调文件列表 GET

2
yudao-server/src/main/resources/application-local.yaml
View File

@ -284,6 +284,8 @@ llm:
login_password: admin
# 创建微调任务 POST
finetuning_create: http://36.103.199.104:5123/llm/finetuning
# 日志获取
finetuning_log: http://36.103.199.104:5123/llm/get_log
# 微调任务详情 GET
finetuning_detail: http://36.103.199.104:9000/api/finetuning
# 微调任务取消

2
yudao-server/src/main/resources/application-prod.yaml
View File

@ -286,6 +286,8 @@ llm:
finetuning_create: http://36.103.199.104:5123/llm/finetuning
# 微调任务详情 GET
finetuning_detail: http://36.133.1.230:8080/api/finetuning
# 日志获取
finetuning_log: http://36.103.199.104:5123/llm/get_log
# 微调任务取消
finetuning_cancel: http://36.103.199.104:9000/api/finetuning/%s/cancel
# 微调文件列表 GET