DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
sunxiqing 2025-01-06 18:36:53 +08:00
commit ba1a80a3bf
6 changed files with 153 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
yudao-module-infra/yudao-module-infra-api/src/main/java/cn/iocoder/yudao/module/infra/enums/DictTypeConstants.java
View File

@ -16,5 +16,6 @@ public interface DictTypeConstants {
String BOOLEAN_STRING = "infra_boolean_string"; // Boolean 是否类型
String OPERATE_TYPE = "infra_operate_type"; // 操作类型
String LABEL_STATUS = "llm_label_status"; // 标签状态
}

21
yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/FineTuningTaskStatusConstants.java Normal file
View File

@ -0,0 +1,21 @@
package cn.iocoder.yudao.module.llm.enums;
import java.util.HashMap;
import java.util.Map;
public class FineTuningTaskStatusConstants {
public final static Map<String, Integer> STATUS_MAP = new HashMap<>();
static {
STATUS_MAP.put("waiting", 3);
STATUS_MAP.put("failed", 4);
STATUS_MAP.put("success", 2);
STATUS_MAP.put("running", 1);
STATUS_MAP.put("cancel", 0);
}
public final static Integer getStatus(String status) {
return STATUS_MAP.get(status);
}
}

6
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/label/vo/LabelRespVO.java
View File

@ -1,5 +1,8 @@
package cn.iocoder.yudao.module.llm.controller.admin.label.vo;
import cn.iocoder.yudao.framework.excel.core.annotations.DictFormat;
import cn.iocoder.yudao.framework.excel.core.convert.DictConvert;
import cn.iocoder.yudao.module.infra.enums.DictTypeConstants;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -27,7 +30,8 @@ public class LabelRespVO {
private LocalDateTime createTime;
@Schema(description = "使用状态（0启用 1停用），使用字典（llm_label_status）", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@ExcelProperty("使用状态（0启用 1停用），使用字典（llm_label_status）")
@ExcelProperty(value = "使用状态",converter = DictConvert.class)
@DictFormat(DictTypeConstants.LABEL_STATUS)
private Integer status;
@Schema(description = "标签排序")

12
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java
View File

@ -11,6 +11,7 @@ import cn.iocoder.yudao.module.llm.dal.dataobject.finetuningtask.FineTuningTaskD
import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper;
import cn.iocoder.yudao.module.llm.enums.FinetuningTaskStatusEnum;
import cn.iocoder.yudao.module.llm.service.async.AsyncFineTuningTaskService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
@ -152,12 +153,9 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
@Override
public void startFineTuningTask(Long id) {
FineTuningTaskDO fineTuningTaskDO = validateFineTuningTaskExists(id);
fineTuningTaskMapper.stopStartTask(id,1);
//todo 调用模型服务，开启调优任务
fineTuningTaskMapper.stopStartTask(id, FinetuningTaskStatusEnum.WAITING.getStatus());
//调用模型服务，开启调优任务
asyncFineTuningTaskService.startFineTuningTask(fineTuningTaskDO);
fineTuningTaskMapper.stopStartTask(id,2);
}
@Override
@ -169,12 +167,10 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
@Override
public void stopFineTuningTask(Long id) {
FineTuningTaskDO fineTuningTaskDO = validateFineTuningTaskExists(id);
fineTuningTaskMapper.stopStartTask(id,1);
fineTuningTaskMapper.stopStartTask(id, FinetuningTaskStatusEnum.WAITING.getStatus());
//todo 调用模型服务，停止调优任务
asyncFineTuningTaskService.stopFineTuningTask(fineTuningTaskDO);
fineTuningTaskMapper.stopStartTask(id,0);
}

45
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java Normal file
View File

@ -0,0 +1,45 @@
package cn.iocoder.yudao.module.llm.service.finetuningtask;
import cn.iocoder.yudao.module.llm.dal.dataobject.finetuningtask.FineTuningTaskDO;
import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper;
import cn.iocoder.yudao.module.llm.enums.FineTuningTaskStatusConstants;
import cn.iocoder.yudao.module.llm.enums.FinetuningTaskStatusEnum;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO;
import com.esotericsoftware.minlog.Log;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
@Component
public class FineTuningTaskSyncService {
@Resource
TrainHttpService trainHttpService;
@Resource
FineTuningTaskMapper fineTuningTaskMapper;
@Scheduled(cron ="30 * * * * ?")
public void updateFineTuningTaskStatus() {
Log.info("FineTuningTaskSync 定时任务启动");
List<FineTuningTaskDO> fineTuningTaskDOList = fineTuningTaskMapper.selectList();
for (FineTuningTaskDO fineTuningTaskDO : fineTuningTaskDOList) {
if(Objects.equals(fineTuningTaskDO.getStatus(), FinetuningTaskStatusEnum.TRAINING.getStatus())
|| Objects.equals(fineTuningTaskDO.getStatus(), FinetuningTaskStatusEnum.WAITING.getStatus())){
AigcFineTuningDetailRespVO resp = trainHttpService.finetuningDetail(new HashMap<>(), fineTuningTaskDO.getJobId());
Integer status = FineTuningTaskStatusConstants.getStatus(resp.getTrainStatus());
if(status != null){
FineTuningTaskDO updateObj = new FineTuningTaskDO();
updateObj.setId(fineTuningTaskDO.getId());
updateObj.setStatus(status);
fineTuningTaskMapper.updateById(updateObj);
}
}
}
}
}

77
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningDetailRespVO.java
View File

@ -1,4 +1,81 @@
package cn.iocoder.yudao.module.llm.service.http.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
import java.util.Date;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class AigcFineTuningDetailRespVO {
private int id;
private String jobId;
private String baseModel;
private int trainEpoch;
private String trainStatus;
private String trainDuration;
private int process;
private String fineTunedModel;
private String remark;
private String finishedAt;
private ZonedDateTime createdAt;
private String trainPublisher;
private String trainLog;
private String errorMessage;
private boolean lora;
private TrainAnalysis trainAnalysis;
private String suffix;
private int modelMaxLength;
private int trainBatchSize;
private String learningRate;
private String fileUrl;
private String fileId;
private String startTrainTime;
private int procPerNode;
private int evalBatchSize;
private int accumulationSteps;
private String scenario;
private Diagnosis diagnosis;
// Nested classes for TrainAnalysis and Diagnosis
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class TrainAnalysis {
private Epoch epoch;
private Loss loss;
private LearningRate learningRate;
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class Epoch {
private List<Integer> list;
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class Loss {
private List<Double> list;
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class LearningRate {
private List<Double> list;
}
public static class Diagnosis {
// Add fields, getters, and setters as needed
}
}