diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/vo/ModelServiceSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/vo/ModelServiceSaveReqVO.java index 277d00be0..d9eef4bad 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/vo/ModelServiceSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/vo/ModelServiceSaveReqVO.java @@ -13,7 +13,7 @@ public class ModelServiceSaveReqVO { private Long id; @Schema(description = "服务名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "芋艿") - @NotEmpty(message = "服务名称不能为空") +// @NotEmpty(message = "服务名称不能为空") private String serviceName; @Schema(description = "微调任务ID") @@ -29,7 +29,7 @@ public class ModelServiceSaveReqVO { private Integer gpuCount; @Schema(description = "模型服务状态,使用字典(llm_model_status)", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") - @NotNull(message = "模型服务状态,使用字典(llm_model_status)不能为空") +// @NotNull(message = "模型服务状态,使用字典(llm_model_status)不能为空") private Integer status; private Long jobId; @@ -45,4 +45,10 @@ public class ModelServiceSaveReqVO { */ @Schema(description = "模型版本号", example = "1.0.0") private String version; + + /** + * 模型url + */ + @Schema(description = "模型url", example = "www.sdada.com") + private String modelUrl; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/modelservice/ModelServiceDO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/modelservice/ModelServiceDO.java index 24347f8e2..95d6adac8 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/modelservice/ModelServiceDO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/modelservice/ModelServiceDO.java @@ -72,4 +72,6 @@ public class ModelServiceDO extends BaseDO { * 模型版本号 */ private String version; + + private String modelUrl; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelAccessManualService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelAccessManualService.java index 7dc9016ed..12a5a893a 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelAccessManualService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelAccessManualService.java @@ -103,10 +103,12 @@ public class AsyncModelAccessManualService { messages.add(message); String modelName = ""; + String modelUrl = ""; if(modelAssessTaskAuto.getModelType() == 0){ ModelServiceDO modelServiceDO = modelServiceMapper.selectById(modelAssessTaskAuto.getModelService()); if (modelServiceDO != null){ modelName = modelServiceDO.getBaseModelName(); + modelUrl = modelServiceDO.getModelUrl(); } }else{ BaseModelDO baseModelDO = baseModelMapper.selectById(modelAssessTaskAuto.getModelService()); @@ -118,7 +120,7 @@ public class AsyncModelAccessManualService { ModelCompletionsReqVO modelCompletionsReqVO = new ModelCompletionsReqVO(); modelCompletionsReqVO.setMessages(messages); modelCompletionsReqVO.setModel(modelName); - ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelCompletionsReqVO); + ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelUrl,modelCompletionsReqVO); String prompt = modelCompletionsRespVO.getAnswer(); String res = trainHttpService.autoEvaluation(prompt, datasetPrompt); @@ -199,7 +201,9 @@ public class AsyncModelAccessManualService { try { if (!CollectionUtils.isAnyEmpty(datasetQuestionList)){ String modelName; + String modelUrl; if (task.getModelType()==1){ + modelUrl = ""; BaseModelDO baseModelDO = baseModelMapper.selectById(task.getModelService()); if (baseModelDO != null){ modelName = baseModelDO.getAigcModelName(); @@ -210,7 +214,9 @@ public class AsyncModelAccessManualService { ModelServiceDO modelServiceDO = modelServiceMapper.selectById(task.getModelService()); if (modelServiceDO != null){ modelName = modelServiceDO.getBaseModelName(); + modelUrl = modelServiceDO.getModelUrl(); } else { + modelUrl = ""; modelName = ""; } } @@ -225,7 +231,7 @@ public class AsyncModelAccessManualService { List messages = new ArrayList<>(); messages.add(message); modelCompletionsReqVO.setMessages(messages); - ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelCompletionsReqVO); + ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelUrl,modelCompletionsReqVO); if (modelCompletionsRespVO != null){ ManualModelAnswerDO manualModelAnswerDO = new ManualModelAnswerDO(); manualModelAnswerDO.setCreator(task.getCreator()); diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelServiceService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelServiceService.java index 634652e95..1d0a42cc2 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelServiceService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelServiceService.java @@ -6,10 +6,7 @@ import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper; import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper; import cn.iocoder.yudao.module.llm.dal.mysql.modelservice.ModelServiceMapper; import cn.iocoder.yudao.module.llm.service.http.TrainHttpService; -import cn.iocoder.yudao.module.llm.service.http.vo.AIgcModelCreateSaveReq; -import cn.iocoder.yudao.module.llm.service.http.vo.AigcModelCreateRespVO; -import cn.iocoder.yudao.module.llm.service.http.vo.AigcModelDeploySaveReq; -import cn.iocoder.yudao.module.llm.service.http.vo.AigcRespVO; +import cn.iocoder.yudao.module.llm.service.http.vo.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Async; @@ -23,6 +20,7 @@ import java.util.HashMap; public class AsyncModelServiceService { private static final Logger log = LoggerFactory.getLogger(AsyncModelServiceService.class); + private static final String DEFAULT_MODEL_URL_SUFFIX = "/v1/chat/completions"; @Resource private FineTuningTaskMapper fineTuningTaskMapper; @@ -104,15 +102,11 @@ public class AsyncModelServiceService { */ ModelServiceDO modelServiceDO = modelServiceMapper.selectById(updateObj.getId()); AigcModelDeploySaveReq aigcModelDeploySaveReq = new AigcModelDeploySaveReq(modelServiceDO.getBaseModelName(), - 1, - updateObj.getGpuCount(), - "cpu", - "cpu-aigc-model", - 0, - "", - "float16", 1); - AigcRespVO aigcRespVO = trainHttpService.modelDeploy(new HashMap<>(), aigcModelDeploySaveReq); + "gpu"); + ModelDeployRespVO modelDeployRespVO = trainHttpService.modelDeploy(new HashMap<>(), aigcModelDeploySaveReq); updateObj.setStatus(1); + updateObj.setJobId(modelDeployRespVO.getId()); + updateObj.setModelUrl(modelDeployRespVO.getPort()+DEFAULT_MODEL_URL_SUFFIX); modelServiceMapper.updateById(updateObj); }catch(Exception e){ // updateObj.setStatus(3); @@ -126,7 +120,7 @@ public class AsyncModelServiceService { try { ModelServiceDO modelServiceDO = modelServiceMapper.selectById(updateObj.getId()); // Thread.sleep(30000); - AigcRespVO aigcRespVO = trainHttpService.modelUndeploy(new HashMap<>(), modelServiceDO.getBaseModelName()); + AigcRespVO aigcRespVO = trainHttpService.modelUndeploy(new HashMap<>(), modelServiceDO.getJobId()); if (aigcRespVO.isSuccess()){ updateObj.setStatus(0); }else { diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java index 65cdc7daf..1ac5d89ee 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java @@ -163,6 +163,7 @@ public class ConversationServiceImpl implements ConversationService { chatReqVO.setUuid(UUID.randomUUID().toString()); } String model = null; + String selfModelUrl = ""; if (Objects.equals(1, chatReqVO.getModelType())) { // 预制模型 BaseModelDO baseModelDO = baseModelService.getBaseModel(chatReqVO.getModelId()); @@ -177,6 +178,7 @@ public class ConversationServiceImpl implements ConversationService { throw exception(MODEL_SERVICE_NOT_EXISTS); } model = modelServiceDO.getBaseModelName(); + selfModelUrl = modelServiceDO.getModelUrl(); }else { throw exception(BASE_MODEL_NOT_EXISTS); } @@ -251,7 +253,7 @@ public class ConversationServiceImpl implements ConversationService { modelCompletionsReqVO.setMessages(messages); // baseModel 的 aigcModelName 为aigc中的模型名称 modelCompletionsReqVO.setModel(model); - ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelCompletionsReqVO); + ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(selfModelUrl,modelCompletionsReqVO); if (modelCompletionsRespVO == null) { throw exception(MODEL_COMPLETIONS_ERROR); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/ModelService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/ModelService.java index 9c261c25b..a9ad51fe6 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/ModelService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/ModelService.java @@ -60,13 +60,20 @@ public class ModelService { * @param req * @return */ - public ModelCompletionsRespVO modelCompletions(ModelCompletionsReqVO req) { + public ModelCompletionsRespVO modelCompletions(String url,ModelCompletionsReqVO req) { if (StringUtils.isBlank(req.getModel())) { req.setModel(DEFAULT_MODEL_ID); } - log.info("url: {}", llmBackendProperties.getModelCompletions()); + log.info("request: {}", req); - String result = HttpUtils.post(llmBackendProperties.getModelCompletions(), null, JSON.toJSONString(req)); + String result; + if (StringUtils.isBlank(url)){ + log.info("url: {}", llmBackendProperties.getModelCompletions()); + result = HttpUtils.post(llmBackendProperties.getModelCompletions(), null, JSON.toJSONString(req)); + }else { + log.info("url: {}", url); + result = HttpUtils.post(url, null, JSON.toJSONString(req)); + } log.info("response: {}", result); if (StringUtils.isBlank(result)) { return null; diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java index 45da52a6a..a22c522ba 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java @@ -238,18 +238,18 @@ public class TrainHttpService { return res; } - public AigcRespVO modelDeploy(Map headers,AigcModelDeploySaveReq req){ + public ModelDeployRespVO modelDeploy(Map headers,AigcModelDeploySaveReq req){ login(headers); String modelDeploy = llmBackendProperties.getModelDeploy(); String res = HttpUtils.post(modelDeploy, headers, JSON.toJSONString(req)); log.info(" modelDeploy:{}", res); - AigcRespVO aigcRespVO = JSON.parseObject(res, AigcRespVO.class); - return aigcRespVO; + ModelDeployRespVO modelDeployRespVO = JSON.parseObject(res.getBytes(), ModelDeployRespVO.class); + return modelDeployRespVO; } - public AigcRespVO modelUndeploy(Map headers, String baseModelName) { + public AigcRespVO modelUndeploy(Map headers, Long deployId) { login(headers); String modelDeploy = llmBackendProperties.getModelUndeploy(); - String res = HttpUtils.post(modelDeploy + baseModelName, headers,""); + String res = HttpUtils.post(modelDeploy + deployId, headers,""); log.info(" modelDeploy:{}", res); AigcRespVO aigcRespVO = JSON.parseObject(res, AigcRespVO.class); log.info(" modelDeploy:{}", aigcRespVO); diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeploySaveReq.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeploySaveReq.java index d0e4e1e27..922295c31 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeploySaveReq.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeploySaveReq.java @@ -20,15 +20,8 @@ public class AigcModelDeploySaveReq { */ private String model; - private Integer cpu; - private Integer gpu; /** * 类型:cpu/gpu */ private String inferredType; - private String label; - private Integer maxGpuMemory; - private String modelWorker; - private String quantization; - private Integer replicas; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeployVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeployVO.java index b57762bf0..eb7d17a3e 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeployVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/AigcModelDeployVO.java @@ -60,4 +60,6 @@ public class AigcModelDeployVO { private String label; private Integer cpu; private Integer vllm; + private String host; + private String port; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceTaskSyncService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceTaskSyncService.java index a0842e722..8d700f699 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceTaskSyncService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceTaskSyncService.java @@ -62,7 +62,7 @@ public class ModelServiceTaskSyncService { // 使用 TypeReference 解析 JSON 字符串为 List try { - String query = "?filter={\"deploy_name\":"+modelServiceDO.getBaseModelName()+"}"; + String query = "?filter={\"id\":"+jobid+"}"; String res = trainHttpService.modelTableQuery(new HashMap<>(), "model_deploy",query); log.info("获取 aigc model_deploy 表数据 info {}",res); ObjectMapper mapper = new ObjectMapper(); diff --git a/yudao-server/src/main/resources/application-dev.yaml b/yudao-server/src/main/resources/application-dev.yaml index f018e704f..f5e16df29 100644 --- a/yudao-server/src/main/resources/application-dev.yaml +++ b/yudao-server/src/main/resources/application-dev.yaml @@ -254,15 +254,15 @@ llm: # 开始部署 model_deploy: http://36.103.199.104:5123/llm/deploy # 取消部署 - model_undeploy: http://36.103.199.104:5123/llm/stop_task?model_name= + model_undeploy: http://36.103.199.104:5123/llm/deploy/stop?deploy_id= # 微调文件上传 aigc_file_upload: http://36.103.199.104:9000/api/files # #### 大模型对话 # 基础模型列表 GET - base_model_list: http://36.103.199.104:9997/model/v1/models + base_model_list: http://36.103.199.104:30000/model/v1/models # 模型对话 POST - model_completions: http://36.103.199.104:9997/v1/chat/completions + model_completions: http://36.103.199.104:30000/v1/chat/completions # aigc表数据查询接口 table_data_query: http://36.103.199.104:5123/table/%s # aigc模型推理 diff --git a/yudao-server/src/main/resources/application-local.yaml b/yudao-server/src/main/resources/application-local.yaml index 5c4fbfb58..274b72584 100644 --- a/yudao-server/src/main/resources/application-local.yaml +++ b/yudao-server/src/main/resources/application-local.yaml @@ -297,12 +297,12 @@ llm: # 开始部署 model_deploy: http://36.103.199.104:5123/llm/deploy # 取消部署 - model_undeploy: http://36.103.199.104:5123/llm/stop_task?model_name= + model_undeploy: http://36.103.199.104:5123/llm/deploy/stop?deploy_id= #### 大模型对话 # 模型列表 GET - base_model_list: http://36.103.199.104:9997/model/v1/models + base_model_list: http://36.103.199.104:30000/model/v1/models # 模型对话 POST - model_completions: http://36.103.199.104:9997/v1/chat/completions + model_completions: http://36.103.199.104:30000/v1/chat/completions # aigc表数据查询接口 table_data_query: http://36.103.199.104:5123/table/%s # aigc模型推理 diff --git a/yudao-server/src/main/resources/application-prod.yaml b/yudao-server/src/main/resources/application-prod.yaml index 399c8639f..d8378b05c 100644 --- a/yudao-server/src/main/resources/application-prod.yaml +++ b/yudao-server/src/main/resources/application-prod.yaml @@ -297,12 +297,12 @@ llm: # 开始部署 model_deploy: http://36.103.199.104:5123/llm/deploy # 取消部署 - model_undeploy: http://36.103.199.104:5123/llm/stop_task?model_name= + model_undeploy: http://36.103.199.104:5123/llm/deploy/stop?deploy_id= #### 大模型对话 # 模型列表 GET - base_model_list: http://36.103.199.104:9997/model/v1/models + base_model_list: http://36.103.199.104:30000/model/v1/models # 模型对话 POST - model_completions: http://36.103.199.104:9997/v1/chat/completions + model_completions: http://36.103.199.104:30000/v1/chat/completions # aigc表数据查询接口 table_data_query: http://36.133.1.230:5123/table/%s # aigc模型推理