From e7d5551a0c8776add98ec57f5481641a1d0da941 Mon Sep 17 00:00:00 2001 From: ire <931903008@qq.com> Date: Sat, 8 Feb 2025 17:10:19 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A3=80=E6=9F=A5=E7=82=B9=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vo/FineTuningTaskRespVO.java | 5 +- .../vo/ModelAssessTaskManualSaveReqVO.java | 2 +- .../finetuningtask/FineTuningTaskDO.java | 8 ++++ .../backend/config/LLMBackendProperties.java | 2 + .../FineTuningTaskServiceImpl.java | 2 + .../FineTuningTaskSyncService.java | 47 +++++++++++++++++++ .../llm/service/http/TrainHttpService.java | 11 +++++ .../src/main/resources/application-dev.yaml | 2 + .../src/main/resources/application-local.yaml | 6 ++- .../src/main/resources/application-prod.yaml | 5 +- 10 files changed, 84 insertions(+), 6 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/vo/FineTuningTaskRespVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/vo/FineTuningTaskRespVO.java index 2124d9e62..dc8051704 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/vo/FineTuningTaskRespVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/finetuningtask/vo/FineTuningTaskRespVO.java @@ -1,6 +1,7 @@ package cn.iocoder.yudao.module.llm.controller.admin.finetuningtask.vo; import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO; +import com.alibaba.fastjson.JSONObject; import io.swagger.v3.oas.annotations.media.Schema; import lombok.*; import java.util.*; @@ -139,4 +140,6 @@ public class FineTuningTaskRespVO { @Schema(description = "模型广场信息") private AigcFineTuningDetailRespVO detailRespVO; -} \ No newline at end of file + private JSONObject checkPointData; + +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelassesstaskmanual/vo/ModelAssessTaskManualSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelassesstaskmanual/vo/ModelAssessTaskManualSaveReqVO.java index 6363c7942..3c1cd2ce0 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelassesstaskmanual/vo/ModelAssessTaskManualSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelassesstaskmanual/vo/ModelAssessTaskManualSaveReqVO.java @@ -40,4 +40,4 @@ public class ModelAssessTaskManualSaveReqVO { @Schema(description ="模型类型（0普通，1官方）") private Integer modelType; -} \ No newline at end of file +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java index 199e77121..348562494 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/finetuningtask/FineTuningTaskDO.java @@ -150,4 +150,12 @@ public class FineTuningTaskDO extends BaseDO { private Long jobModelListId; // 日志文件地址 private String trainLog; + + //检查点文件地址 + private String checkPointFilePath; + + //检查点数据内容 + private String checkPointData; + + } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java index a214408ee..ea1129783 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java @@ -107,4 +107,6 @@ public class LLMBackendProperties { * 知识库向量嵌入 */ private String embed; + + private String checkFileList; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java index a571b2c91..162b0aa04 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java @@ -17,6 +17,7 @@ import cn.iocoder.yudao.module.llm.service.async.AsyncFineTuningTaskService; import cn.iocoder.yudao.module.llm.service.http.TrainHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO; import cn.iocoder.yudao.module.llm.service.modelservice.ModelServiceService; +import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; @@ -176,6 +177,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService { AigcFineTuningDetailRespVO aigcFineTuningDetailRespVO = trainHttpService.finetuningDetail(new HashMap<>(), fineTuningTaskDO.getJobId()); result.setDetailRespVO(aigcFineTuningDetailRespVO); } + result.setCheckPointData(JSON.parseObject(fineTuningTaskDO.getCheckPointData())); return result; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java index cdaf37e47..ee494e37c 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskSyncService.java @@ -15,6 +15,13 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; import java.util.*; @Component @@ -86,6 +93,44 @@ public class FineTuningTaskSyncService { }catch (Exception e){ log.error(" error {}",e.getMessage()); } + + try { + //获取检查点信息 + String checkFileList = trainHttpService.getCheckFileList(jobModelName); + String checkpoint = ""; + List fileList = JSONArray.parseArray(checkFileList,String.class); + for (String s : fileList) { + if(s.contains("checkpoint")){ + checkpoint = s; + } + } + String filePath = "/" + checkpoint + "/trainer_state.json"; + String fileUrl = trainHttpService.getCheckFile(jobModelName, filePath); + try { + URL url = new URL(fileUrl); + URLConnection urlConnection = url.openConnection(); + InputStream inputStream = urlConnection.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + StringBuilder content = new StringBuilder(); + while ((line = reader.readLine()) != null) { + content.append(line + "\n"); + } + reader.close(); + System.out.println(content.toString()); + updateObj.setCheckPointData(content.toString()); + updateObj.setCheckPointFilePath(fileUrl); + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + + }catch (Exception e){ + log.error(" error {}",e.getMessage()); + } + fineTuningTaskMapper.updateById(updateObj); } @@ -98,4 +143,6 @@ public class FineTuningTaskSyncService { } } + + } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java index 8c8dd40ef..45da52a6a 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java @@ -324,4 +324,15 @@ public class TrainHttpService { String finetuningLog = llmBackendProperties.getFinetuningLog(); return HttpUtils.get(finetuningLog+params, null); } + + public String getCheckFileList(String name){ + String checkFileList = llmBackendProperties.getCheckFileList(); + return HttpUtils.get(checkFileList + name, null); + } + + public String getCheckFile(String name,String path){ + String checkFileList = llmBackendProperties.getCheckFileList(); + return checkFileList + name + "&&file_path=" + path; + + } } diff --git a/yudao-server/src/main/resources/application-dev.yaml b/yudao-server/src/main/resources/application-dev.yaml index bfc253543..5370bc5cb 100644 --- a/yudao-server/src/main/resources/application-dev.yaml +++ b/yudao-server/src/main/resources/application-dev.yaml @@ -282,6 +282,8 @@ llm: # 文生图 text_to_image: http://36.103.199.104:5123/generate-image + check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name= + # 知识库向量嵌入 embed: http://36.103.199.104:8123/embed --- #################### iot相关配置 TODO 芋艿：再瞅瞅 #################### diff --git a/yudao-server/src/main/resources/application-local.yaml b/yudao-server/src/main/resources/application-local.yaml index 260042e7b..f33ed6541 100644 --- a/yudao-server/src/main/resources/application-local.yaml +++ b/yudao-server/src/main/resources/application-local.yaml @@ -321,12 +321,14 @@ llm: optimize_prompt: http://36.103.199.104:5123/optimize-prompt auto_evaluation: http://36.103.199.104:5123/llm-eval - # 文生图 text_to_image: http://36.103.199.104:5123/generate-image - # 知识库向量嵌入 embed: http://36.103.199.104:8123/embed + + check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name= + + --- #################### iot相关配置 TODO 芋艿：再瞅瞅 #################### iot: emq: diff --git a/yudao-server/src/main/resources/application-prod.yaml b/yudao-server/src/main/resources/application-prod.yaml index 88d63dded..526054b2b 100644 --- a/yudao-server/src/main/resources/application-prod.yaml +++ b/yudao-server/src/main/resources/application-prod.yaml @@ -321,12 +321,13 @@ llm: optimize_prompt: http://36.133.1.230:5123/optimize-prompt auto_evaluation: http://36.133.1.230:5123/llm-eval - # 文生图 text_to_image: http://36.133.1.230:5123/generate-image - # 知识库向量嵌入 embed: http://36.103.199.104:8123/embed + + check_file_list: http://36.103.199.104:5123/1lm/finetuning/checkpoints?model_name= + --- #################### iot相关配置 TODO 芋艿：再瞅瞅 #################### iot: emq: