From ec5ccfd0699467f5d286a3bb652496c56c15a314 Mon Sep 17 00:00:00 2001 From: zhangtao Date: Mon, 6 Jan 2025 18:05:51 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BE=AE=E8=B0=83=E4=BB=BB=E5=8A=A1API?= =?UTF-8?q?=E5=AF=B9=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../enums/FineTuningTaskStatusConstants.java | 21 +++++ .../FineTuningTaskSyncService.java | 45 +++++++++++ .../http/vo/AigcFineTuningDetailRespVO.java | 79 +++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/FineTuningTaskStatusConstants.java create mode 100644 yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java diff --git a/yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/FineTuningTaskStatusConstants.java b/yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/FineTuningTaskStatusConstants.java new file mode 100644 index 000000000..cbcf6a8b5 --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/FineTuningTaskStatusConstants.java @@ -0,0 +1,21 @@ +package cn.iocoder.yudao.module.llm.enums; + +import java.util.HashMap; +import java.util.Map; + +public class FineTuningTaskStatusConstants { + + public final static Map STATUS_MAP = new HashMap<>(); + + static { + STATUS_MAP.put("waiting", 3); + STATUS_MAP.put("failed", 4); + STATUS_MAP.put("success", 2); + STATUS_MAP.put("running", 1); + STATUS_MAP.put("cancel", 0); + } + + public final static Integer getStatus(String status) { + return STATUS_MAP.get(status); + } +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java new file mode 100644 index 000000000..7db903b1a --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java @@ -0,0 +1,45 @@ +package cn.iocoder.yudao.module.llm.service.finetuningtask; + +import cn.iocoder.yudao.module.llm.dal.dataobject.finetuningtask.FineTuningTaskDO; +import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper; +import cn.iocoder.yudao.module.llm.enums.FineTuningTaskStatusConstants; +import cn.iocoder.yudao.module.llm.enums.FinetuningTaskStatusEnum; +import cn.iocoder.yudao.module.llm.service.http.TrainHttpService; +import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO; +import com.esotericsoftware.minlog.Log; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; + +@Component +public class FineTuningTaskSyncService { + + @Resource + TrainHttpService trainHttpService; + + @Resource + FineTuningTaskMapper fineTuningTaskMapper; + + @Scheduled(cron ="30 * * * * ?") + public void updateFineTuningTaskStatus() { + Log.info("FineTuningTaskSync 定时任务启动"); + List fineTuningTaskDOList = fineTuningTaskMapper.selectList(); + for (FineTuningTaskDO fineTuningTaskDO : fineTuningTaskDOList) { + if(Objects.equals(fineTuningTaskDO.getStatus(), FinetuningTaskStatusEnum.TRAINING.getStatus()) + || Objects.equals(fineTuningTaskDO.getStatus(), FinetuningTaskStatusEnum.WAITING.getStatus())){ + AigcFineTuningDetailRespVO resp = trainHttpService.finetuningDetail(new HashMap<>(), fineTuningTaskDO.getJobId()); + Integer status = FineTuningTaskStatusConstants.getStatus(resp.getTrainStatus()); + if(status != null){ + FineTuningTaskDO updateObj = new FineTuningTaskDO(); + updateObj.setId(fineTuningTaskDO.getId()); + updateObj.setStatus(status); + fineTuningTaskMapper.updateById(updateObj); + } + } + } + } +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningDetailRespVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningDetailRespVO.java index 25e9bb180..b43b59328 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningDetailRespVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcFineTuningDetailRespVO.java @@ -1,4 +1,83 @@ package cn.iocoder.yudao.module.llm.service.http.vo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor public class AigcFineTuningDetailRespVO { + + private int id; + private String jobId; + private String baseModel; + private int trainEpoch; + private String trainStatus; + private String trainDuration; + private int process; + private String fineTunedModel; + private String remark; + private String finishedAt; + private Date createdAt; + private String trainPublisher; + private String trainLog; + private String errorMessage; + private boolean lora; + private TrainAnalysis trainAnalysis; + private String suffix; + private int modelMaxLength; + private int trainBatchSize; + private String learningRate; + private String fileUrl; + private String fileId; + private String startTrainTime; + private int procPerNode; + private int evalBatchSize; + private int accumulationSteps; + private String scenario; + private Diagnosis diagnosis; + + + // Nested classes for TrainAnalysis and Diagnosis + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class TrainAnalysis { + private Epoch epoch; + private Loss loss; + private LearningRate learningRate; + + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Epoch { + private List list; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Loss { + private List list; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class LearningRate { + private List list; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Diagnosis { + // Add fields, getters, and setters as needed + } }