DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
sunxiqing 2025-01-06 16:43:22 +08:00
commit 5fedd8dcd9
13 changed files with 151 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/ErrorCodeConstants.java
View File

@ -92,6 +92,7 @@ public interface ErrorCodeConstants {
ErrorCode PROMPT_TEMPLATES_TAGS_BACKUP_NOT_EXISTS = new ErrorCode(10037, "模板信息不存在");
ErrorCode DATA_PROCESS_TASK_NAME_NOT_EXISTS = new ErrorCode(10038, "数据处理任务名称已存在");
ErrorCode FINE_TUNING_TASK_NAME_NOT_EXISTS = new ErrorCode(10039, "模型调优任务名称已存在");
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/basemodel/vo/BaseModelPageReqVO.java
View File

@ -49,4 +49,6 @@ public class BaseModelPageReqVO extends PageParam {
@Schema(description = "代理")
private String proxy;
@Schema(description = "aigc平台id")
private Long aigcId;
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/basemodel/vo/BaseModelRespVO.java
View File

@ -58,4 +58,7 @@ public class BaseModelRespVO {
@Schema(description = "代理")
private String proxy;
@Schema(description = "aigc平台id")
private Long aigcId;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/basemodel/vo/BaseModelSaveReqVO.java
View File

@ -48,4 +48,6 @@ public class BaseModelSaveReqVO {
@Schema(description = "代理")
private String proxy;
@Schema(description = "aigc平台id")
private Long aigcId;
}

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/basemodel/BaseModelDO.java
View File

@ -68,4 +68,9 @@ public class BaseModelDO extends BaseDO {
*/
private String proxy;
/**
* augc平台的id
*/
private Long aigcId;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelService.java
View File

@ -54,4 +54,6 @@ public interface BaseModelService {
List<BaseModelDO> getBaseModelList();
BaseModelDO getByAigcId(Long id);
}

27
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelServiceImpl.java
View File

@ -1,22 +1,20 @@
package cn.iocoder.yudao.module.llm.service.basemodel;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.controller.admin.basemodel.vo.BaseModelPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.basemodel.vo.BaseModelSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import org.springframework.validation.annotation.Validated;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import cn.iocoder.yudao.module.llm.controller.admin.basemodel.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
import javax.annotation.Resource;
import java.util.List;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.*;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.BASE_MODEL_NOT_EXISTS;
/**
* 基座模型 Service 实现类
@ -81,4 +79,11 @@ public class BaseModelServiceImpl implements BaseModelService {
return baseModelDOS;
}
@Override
public BaseModelDO getByAigcId(Long id) {
LambdaQueryWrapper<BaseModelDO> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(BaseModelDO::getAigcId,id);
return baseModelMapper.selectOne(queryWrapper);
}
}

68
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java Normal file
View File

@ -0,0 +1,68 @@
package cn.iocoder.yudao.module.llm.service.basemodel;
import cn.hutool.json.JSONUtil;
import cn.iocoder.yudao.module.llm.controller.admin.basemodel.vo.BaseModelSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.service.basemodel.vo.ModelListRes;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.esotericsoftware.minlog.Log;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
@SpringBootApplication
@EnableScheduling
public class BaseModelTaskService {
@Resource
TrainHttpService trainHttpService;
@Resource
BaseModelService baseModelService;
@Scheduled(cron ="0 0 0/1 * * ?")
public void updateBaseModel() {
Log.info("定时任务启动");
String resStr = trainHttpService.modelsList();
Log.info("获取aicg模型列表返回数据内容{}",resStr);
JSONObject jsonObject = JSONObject.parseObject(resStr);
JSONObject data = jsonObject.getJSONObject("data");
JSONArray list = data.getJSONArray("list");
String s = list.toJSONString();
List<ModelListRes> modelListRes = JSONObject.parseArray(s, ModelListRes.class);
for (ModelListRes modelListRe : modelListRes) {
Long id = modelListRe.getId();
BaseModelDO byAigcId = baseModelService.getByAigcId(id);
if(byAigcId == null){
BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO();
baseModelSaveReqVO.setAigcId(id);
baseModelSaveReqVO.setModelName(modelListRe.getModelName());
baseModelSaveReqVO.setModelType(modelListRe.getModelType());
baseModelSaveReqVO.setMaxContextLength(modelListRe.getMaxTokens());
baseModelSaveReqVO.setIsActive(modelListRe.getEnabled());
baseModelSaveReqVO.setIsFinetuned(modelListRe.getIsFineTuning());
baseModelSaveReqVO.setNotes(modelListRe.getRemark());
baseModelSaveReqVO.setParameterCount(1);
baseModelService.createBaseModel(baseModelSaveReqVO);
}else{
BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO();
baseModelSaveReqVO.setId(byAigcId.getId());
baseModelSaveReqVO.setParameterCount(1);
baseModelSaveReqVO.setModelName(modelListRe.getModelName());
baseModelSaveReqVO.setModelType(modelListRe.getModelType());
baseModelSaveReqVO.setMaxContextLength(modelListRe.getMaxTokens());
baseModelSaveReqVO.setIsActive(modelListRe.getEnabled());
baseModelSaveReqVO.setIsFinetuned(modelListRe.getIsFineTuning());
baseModelSaveReqVO.setNotes(modelListRe.getRemark());
baseModelService.updateBaseModel(baseModelSaveReqVO);
}
}
}
}

31
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/vo/ModelListRes.java Normal file
View File

@ -0,0 +1,31 @@
package cn.iocoder.yudao.module.llm.service.basemodel.vo;
import lombok.Data;
@Data
public class ModelListRes {
private Long id;
//模型名称
private String modelName;
//最长上下文长度
private Integer maxTokens;
//模型类型
private String modelType;
//备注
private String remark;
//是否启用
private Boolean enabled;
//是否微调
private Boolean isFineTuning;
//部署状态
private String deployStatus;
}

16
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/finetuningtask/FineTuningTaskServiceImpl.java
View File

@ -25,7 +25,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.FINE_TUNING_TASK_NOT_EXISTS;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.*;
/**
@ -49,6 +49,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
@Override
public Long createFineTuningTask(FineTuningTaskSaveReqVO createReqVO) {
// 插入
validateFineTuningNameTaskExists(createReqVO);
FineTuningTaskDO fineTuningTask = BeanUtils.toBean(createReqVO, FineTuningTaskDO.class);
fineTuningTaskMapper.insert(fineTuningTask);
//todo 调用模型服务，创建调优任务
@ -61,6 +62,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
public void updateFineTuningTask(FineTuningTaskSaveReqVO updateReqVO) {
// 校验存在
validateFineTuningTaskExists(updateReqVO.getId());
validateFineTuningNameTaskExists(updateReqVO);
// 更新
FineTuningTaskDO updateObj = BeanUtils.toBean(updateReqVO, FineTuningTaskDO.class);
fineTuningTaskMapper.updateById(updateObj);
@ -83,6 +85,18 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
throw exception(FINE_TUNING_TASK_NOT_EXISTS);
}
}
private void validateFineTuningNameTaskExists(FineTuningTaskSaveReqVO reqVO) {
LambdaQueryWrapper<FineTuningTaskDO> wrapper = new LambdaQueryWrapper<FineTuningTaskDO>()
.eq(FineTuningTaskDO::getTaskName, reqVO.getTaskName());
if (reqVO.getId() != null){
wrapper.ne(FineTuningTaskDO::getId, reqVO.getId());
}
List<FineTuningTaskDO> dineTuningTaskDos = fineTuningTaskMapper.selectList(wrapper);
if (CollectionUtils.isNotEmpty(dineTuningTaskDos)){
throw exception(FINE_TUNING_TASK_NAME_NOT_EXISTS);
}
}
@Override
public FineTuningTaskRespVO getFineTuningTask(Long id) {

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java
View File

@ -10,6 +10,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.Map;
/**
@ -80,7 +81,9 @@ public class TrainHttpService {
/**
* 大模型列表 GET
*/
public String modelsList(Map<String, String> headers){
public String modelsList(){
Map<String, String> headers = new HashMap<>();
login(headers);
String modelsList = llmBackendProperties.getModelsList();
String res = HttpUtils.get(modelsList, headers);
return res;

2
yudao-server/src/main/resources/application-dev.yaml
View File

@ -233,7 +233,7 @@ llm:
# 保存标注 POST
annotation_task_save: http://localhost:8123/api/mgr/annotation/task/task-6025001b-692c-44a1-9bc7-2a34bd7c0efe/segment/das-2eedd7bf-3770-4816-a961-b30c446b7a4f/mark
# 大模型列表 GET
models_list: http://localhost:8123/api/models
models_list: http://aigc.xhllm.xinnuojinzhi.com/api/models
# 登录 POST
login: http://aigc.xhllm.xinnuojinzhi.com/api/auth/login
account: http://aigc.xhllm.xinnuojinzhi.com/api/auth/account

2
yudao-server/src/main/resources/application-local.yaml
View File

@ -276,7 +276,7 @@ llm:
# 保存标注 POST
annotation_task_save: http://localhost:8123/api/mgr/annotation/task/task-6025001b-692c-44a1-9bc7-2a34bd7c0efe/segment/das-2eedd7bf-3770-4816-a961-b30c446b7a4f/mark
# 大模型列表 GET
models_list: http://localhost:8123/api/models
models_list: http://aigc.xhllm.xinnuojinzhi.com/api/models
# 登录 POST
login: http://aigc.xhllm.xinnuojinzhi.com/api/auth/login
account: http://aigc.xhllm.xinnuojinzhi.com/api/auth/account