DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into master-wangwei

Browse Source
This commit is contained in:
Edward_89 2024-12-31 16:56:01 +08:00
commit 1a249b49ff
30 changed files with 580 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/DataProcessTaskController.java
View File

@ -75,8 +75,7 @@ public class DataProcessTaskController {
@Operation(summary = "获得数据处理任务分页")
@PreAuthorize("@ss.hasPermission('llm:data-process-task:query')")
public CommonResult<PageResult<DataProcessTaskRespVO>> getDataProcessTaskPage(@Valid DataProcessTaskPageReqVO pageReqVO) {
PageResult<DataProcessTaskDO> pageResult = dataProcessTaskService.getDataProcessTaskPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, DataProcessTaskRespVO.class));
return success(dataProcessTaskService.getDataProcessTaskPage1(pageReqVO));
}
@GetMapping("/export-excel")
@ -92,4 +91,19 @@ public class DataProcessTaskController {
BeanUtils.toBean(list, DataProcessTaskRespVO.class));
}
@PutMapping("/stop")
@Operation(summary = "停止模型调优 —— 微调任务")
@PreAuthorize("@ss.hasPermission('llm:fine-tuning-task:update')")
public CommonResult<Boolean> stopFineTuningTask(@RequestParam("id") Long id) {
dataProcessTaskService.stopFineTuningTask(id);
return success(true);
}
@PutMapping("/reStart")
@Operation(summary = "开始模型调优 —— 微调任务")
@PreAuthorize("@ss.hasPermission('llm:fine-tuning-task:update')")
public CommonResult<Boolean> startFineTuningTask(@RequestParam("id") Long id) {
dataProcessTaskService.reStartFineTuningTask(id);
return success(true);
}
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskPageReqVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import lombok.*;
import java.util.*;
import io.swagger.v3.oas.annotations.media.Schema;
@ -32,6 +33,6 @@ public class DataProcessTaskPageReqVO extends PageParam {
private Long datasetPostId;
@Schema(description = "配置信息")
private String options;
private JSONObject options;
}

9
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskRespVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -38,6 +39,12 @@ public class DataProcessTaskRespVO {
@Schema(description = "配置信息")
@ExcelProperty("配置信息")
private String options;
private JSONObject options;
@Schema(description ="处理前的数据集名称")
private String datasetName;
@Schema(description = "处理后的数据集名称")
private String datasetPostName;
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskSaveReqVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -27,6 +28,6 @@ public class DataProcessTaskSaveReqVO {
private Long datasetPostId;
@Schema(description = "配置信息")
private String options;
private JSONObject options;
}

38
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/datarefluxconfig/DataRefluxConfigController.java
View File

@ -1,7 +1,12 @@
package cn.iocoder.yudao.module.llm.controller.admin.datarefluxconfig;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo.DataRefluxDataRespVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.datarefluxdata.DataRefluxDataDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelservice.ModelServiceDO;
import cn.iocoder.yudao.module.llm.dal.mysql.modelservice.ModelServiceMapper;
import cn.iocoder.yudao.module.llm.service.basemodel.BaseModelService;
import cn.iocoder.yudao.module.llm.service.datarefluxdata.DataRefluxDataService;
import cn.iocoder.yudao.module.llm.service.modelservice.ModelServiceService;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
@ -44,6 +49,8 @@ public class DataRefluxConfigController {
private ModelServiceService modelServiceService;
@Resource
private BaseModelService baseModelService;
@Resource
private DataRefluxDataService dataRefluxDataService;
@PostMapping("/create")
@Operation(summary = "创建数据回流配置")
@ -77,9 +84,17 @@ public class DataRefluxConfigController {
DataRefluxConfigDO dataRefluxConfig = dataRefluxConfigService.getDataRefluxConfig(id);
DataRefluxConfigRespVO bean = BeanUtils.toBean(dataRefluxConfig, DataRefluxConfigRespVO.class);
if (bean.getModelType()==0){
bean.setModelServiceName(modelServiceService.getModelService(bean.getModelServiceId()).getServiceName());
ModelServiceDO modelService = modelServiceService.getModelService(bean.getModelServiceId());
if (modelService==null){
bean.setModelServiceName("");
}
bean.setModelServiceName(modelService.getServiceName());
}else if (bean.getModelType()==1){
bean.setModelServiceName(baseModelService.getBaseModel(bean.getModelServiceId()).getModelName());
BaseModelDO baseModel = baseModelService.getBaseModel(bean.getModelServiceId());
if (baseModel==null){
bean.setModelServiceName("");
}
bean.setModelServiceName(baseModel.getModelName());
}
return success(bean);
}
@ -95,15 +110,30 @@ public class DataRefluxConfigController {
PageResult<DataRefluxConfigRespVO> bean = BeanUtils.toBean(pageResult, DataRefluxConfigRespVO.class);
for (DataRefluxConfigRespVO dataRefluxConfigRespVO : bean.getList()){
if (dataRefluxConfigRespVO.getModelType()==0){
dataRefluxConfigRespVO.setModelServiceName(modelServiceService.getModelService(dataRefluxConfigRespVO.getModelServiceId()).getServiceName());
ModelServiceDO modelService = modelServiceService.getModelService(dataRefluxConfigRespVO.getModelServiceId());
if (modelService==null){
dataRefluxConfigRespVO.setModelServiceName("");
}
dataRefluxConfigRespVO.setModelServiceName(modelService.getServiceName());
}else if (dataRefluxConfigRespVO.getModelType()==1){
dataRefluxConfigRespVO.setModelServiceName(baseModelService.getBaseModel(dataRefluxConfigRespVO.getModelServiceId()).getModelName());
BaseModelDO baseModel = baseModelService.getBaseModel(dataRefluxConfigRespVO.getModelServiceId());
if (baseModel==null){
dataRefluxConfigRespVO.setModelServiceName("");
}
dataRefluxConfigRespVO.setModelServiceName(baseModel.getModelName());
}
}
// PageResult<DataRefluxConfigRespVO> result = bean.setList(bean.getList());
return success(bean);
}
@GetMapping("/review-the-data")
@Operation(summary = "数据回流配置查看数据")
public CommonResult<List<DataRefluxDataRespVO>> reviewTheData(@Valid DataRefluxConfigPageReqVO pageReqVO) {
List<DataRefluxDataDO> list = dataRefluxDataService.getTheDetails(pageReqVO);
return success(BeanUtils.toBean(list, DataRefluxDataRespVO.class));
}
@GetMapping("/export-excel")
@Operation(summary = "导出数据回流配置 Excel")
@PreAuthorize("@ss.hasPermission('llm:data-reflux-config:export')")

9
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/datarefluxconfig/vo/DataRefluxConfigPageReqVO.java
View File

@ -15,6 +15,9 @@ import static cn.iocoder.yudao.framework.common.util.date.DateUtils.FORMAT_YEAR_
@ToString(callSuper = true)
public class DataRefluxConfigPageReqVO extends PageParam {
@Schema(description = "配置ID", example = "11768")
private Long id;
@Schema(description = "模型服务ID", example = "2475")
private Long modelServiceId;
@ -25,4 +28,10 @@ public class DataRefluxConfigPageReqVO extends PageParam {
@DateTimeFormat(pattern = FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND)
private LocalDateTime[] createTime;
@Schema(description = "开始时间")
private LocalDateTime startTime;
@Schema(description = "结束时间")
private LocalDateTime endTime;
}

19
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/datarefluxdata/DataRefluxDataController.java
View File

@ -67,6 +67,7 @@ public class DataRefluxDataController {
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('llm:data-reflux-data:query')")
public CommonResult<DataRefluxDataRespVO> getDataRefluxData(@RequestParam("id") Long id) {
//TODO 查看按钮后面没具体内容
DataRefluxDataDO dataRefluxData = dataRefluxDataService.getDataRefluxData(id);
return success(BeanUtils.toBean(dataRefluxData, DataRefluxDataRespVO.class));
}
@ -79,6 +80,22 @@ public class DataRefluxDataController {
return success(BeanUtils.toBean(pageResult, DataRefluxDataRespVO.class));
}
@GetMapping("/review-the-data")
@Operation(summary = "数据回流配置保存数据")
public void saveTheData(@Valid DataRefluxDataReqVO pageReqVO){
if (pageReqVO.getId() != null){
pageReqVO.setIds(Collections.singletonList(pageReqVO.getId()));
}
//TODO 数据回流配置保存数据
}
@GetMapping("/save-all-the-data")
@Operation(summary = "数据回流配置保存全部数据")
public void saveAllTheData(@Valid DataRefluxDataReqVO pageReqVO){
//TODO 全部保存数据
}
@GetMapping("/export-excel")
@Operation(summary = "导出数据回流 —— 数据 Excel")
@PreAuthorize("@ss.hasPermission('llm:data-reflux-data:export')")
@ -92,4 +109,4 @@ public class DataRefluxDataController {
BeanUtils.toBean(list, DataRefluxDataRespVO.class));
}
}
}

16
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/datarefluxdata/vo/DataRefluxDataReqVO.java Normal file
View File

@ -0,0 +1,16 @@
package cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Schema(description = "管理后台 - 数据回流 —— 保存数据")
@Data
public class DataRefluxDataReqVO {
@Schema(description = "配置ID",example = "16823")
private Long id;
@Schema(description = "多个配置ID",example = "16823")
private List<Long> ids;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/datarefluxdata/vo/DataRefluxDataSaveReqVO.java
View File

@ -21,7 +21,7 @@ public class DataRefluxDataSaveReqVO {
@Schema(description = "是否预知模型（0普通 1官方）", requiredMode = Schema.RequiredMode.REQUIRED, example = "0")
@NotNull(message = "是否预知模型（0普通 1官方）不能为空")
private Long modelType;
private Integer modelType;
@Schema(description = "system")
private String system;

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java
View File

@ -78,7 +78,7 @@ public class KnowledgeBaseController {
}
@GetMapping("/list")
@Operation(summary = "获得知识库分页")
@Operation(summary = "获得知识库列表，下拉菜单")
@PreAuthorize("@ss.hasPermission('llm:knowledge-base:query')")
public CommonResult<List<KnowledgeBaseRespVO>> getKnowledgeBaseList() {
List<KnowledgeBaseDO> list = knowledgeBaseService.getKnowledgeBaseList();

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/datarefluxdata/DataRefluxDataDO.java
View File

@ -38,7 +38,7 @@ public class DataRefluxDataDO extends BaseDO {
/**
* 是否预知模型（0普通 1官方）
*/
private Long modelType;
private Integer modelType;
/**
* system
*/

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataprocesstask/DataProcessTaskMapper.java
View File

@ -8,6 +8,8 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import org.apache.ibatis.annotations.Mapper;
import cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo.*;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
/**
* 数据处理任务 Mapper
@ -27,5 +29,6 @@ public interface DataProcessTaskMapper extends BaseMapperX<DataProcessTaskDO> {
.eqIfPresent(DataProcessTaskDO::getOptions, reqVO.getOptions())
.orderByDesc(DataProcessTaskDO::getId));
}
@Update("update llm_data_process_task set status = #{status} where id = #{id}")
void updateStatus(@Param("id") Long id, @Param("status") int status);
}

16
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/AsyncConfig.java Normal file
View File

@ -0,0 +1,16 @@
package cn.iocoder.yudao.module.llm.framework.backend.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import java.util.concurrent.Executor;
@Configuration
@EnableAsync
public class AsyncConfig implements AsyncConfigurer {
@Override
public Executor getAsyncExecutor() {
return AsyncConfigurer.super.getAsyncExecutor();
}
}

7
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java
View File

@ -59,9 +59,12 @@ public class LLMBackendProperties {
@NotNull(message = "登录 POST")
private String login;
@NotNull(message = "大模型列表 GET")
private String modelList;
@NotNull(message = "基础模型列表 GET")
private String baseModelList;
@NotNull(message = "大模型聊天 POST")
private String modelCompletions;
@NotNull(message = "创建微调任务 POST")
private String finetuningCreate;
}

29
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/MyBatisConfig.java Normal file
View File

@ -0,0 +1,29 @@
package cn.iocoder.yudao.module.llm.framework.backend.config;
import cn.hutool.json.JSONObject;
import cn.iocoder.yudao.module.llm.handler.JSONObjectTypeHandler;
import com.baomidou.mybatisplus.autoconfigure.ConfigurationCustomizer;
import org.apache.ibatis.type.TypeHandlerRegistry;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@MapperScan("cn.iocoder.yudao.module.llm.dal.mysql.dataprocesstask")
public class MyBatisConfig {
@Bean
public JSONObjectTypeHandler jsonObjectTypeHandler() {
return new JSONObjectTypeHandler();
}
@Bean
public ConfigurationCustomizer configurationCustomizer() {
return configuration -> {
TypeHandlerRegistry typeHandlerRegistry = configuration.getTypeHandlerRegistry();
if (!typeHandlerRegistry.hasTypeHandler(JSONObject.class)) {
typeHandlerRegistry.register(JSONObject.class, jsonObjectTypeHandler());
}
};
}
}

35
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/handler/JSONObjectTypeHandler.java Normal file
View File

@ -0,0 +1,35 @@
package cn.iocoder.yudao.module.llm.handler;
import cn.hutool.json.JSONObject;
import org.apache.ibatis.type.BaseTypeHandler;
import org.apache.ibatis.type.JdbcType;
import java.sql.CallableStatement;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class JSONObjectTypeHandler extends BaseTypeHandler<JSONObject> {
@Override
public void setNonNullParameter(PreparedStatement ps, int i, JSONObject parameter, JdbcType jdbcType) throws SQLException {
ps.setString(i, parameter.toString());
}
@Override
public JSONObject getNullableResult(ResultSet rs, String columnName) throws SQLException {
String jsonStr = rs.getString(columnName);
return jsonStr == null ? null : new JSONObject(jsonStr);
}
@Override
public JSONObject getNullableResult(ResultSet rs, int columnIndex) throws SQLException {
String jsonStr = rs.getString(columnIndex);
return jsonStr == null ? null : new JSONObject(jsonStr);
}
@Override
public JSONObject getNullableResult(CallableStatement cs, int columnIndex) throws SQLException {
String jsonStr = cs.getString(columnIndex);
return jsonStr == null ? null : new JSONObject(jsonStr);
}
}

40
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java Normal file
View File

@ -0,0 +1,40 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataprocesstask.DataProcessTaskMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@Service
public class AsyncDataProcessService {
@Resource
private DatasetMapper datasetMapper;
@Resource
private DataProcessTaskMapper dataProcessTaskMapper;
/* @Resource
private */
@Async
public void backups(DataProcessTaskDO dataProcessTask) {
try {
// 判断备份数据集是否存在
if (dataProcessTask.getDatasetPostId() == null){
DatasetDO datasetDO = datasetMapper.selectById(dataProcessTask.getDatasetId());
DatasetDO newData = BeanUtils.toBean(datasetDO, DatasetDO.class);
newData.setId(null);
datasetMapper.insert(newData);
dataProcessTask.setDatasetPostId(newData.getId());
dataProcessTaskMapper.updateById(dataProcessTask);
}
}catch(Exception e){
dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 5);
};
}
}

11
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java
View File

@ -1,8 +1,10 @@
package cn.iocoder.yudao.module.llm.service.conversation;
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo.DataRefluxDataSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.service.basemodel.BaseModelService;
import cn.iocoder.yudao.module.llm.service.datarefluxdata.DataRefluxDataService;
import cn.iocoder.yudao.module.llm.service.http.ModelService;
import cn.iocoder.yudao.module.llm.service.http.vo.ModelCompletionsReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.ModelCompletionsRespVO;
@ -44,6 +46,8 @@ public class ConversationServiceImpl implements ConversationService {
private ModelService modelService;
@Resource
private BaseModelService baseModelService;
@Resource
private DataRefluxDataService dataRefluxDataService;
// 聊天会话历史记录缓存Key
private final static String CHAT_HIStORY_REDIS_KEY = "llm:chat:history";
@ -130,6 +134,13 @@ public class ConversationServiceImpl implements ConversationService {
chatRespVO.setResponse(modelCompletionsRespVO.getAnswer());
// 将聊天记录放入缓存
stringRedisTemplate.opsForList().rightPush(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), JsonUtils.toJsonString(message));
DataRefluxDataSaveReqVO dataRefluxDataSaveReqVO = new DataRefluxDataSaveReqVO();
dataRefluxDataSaveReqVO.setModelServiceId(chatReqVO.getModelId());
dataRefluxDataSaveReqVO.setModelType(chatReqVO.getModelType());
dataRefluxDataSaveReqVO.setPrompt(chatReqVO.getPrompt());
dataRefluxDataSaveReqVO.setResponse(modelCompletionsRespVO.getAnswer());
dataRefluxDataSaveReqVO.setSystem(modelCompletionsRespVO.getSystem());
dataRefluxDataService.saveDataRefluxData(dataRefluxDataSaveReqVO);
return chatRespVO;
}
}

10
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataprocesstask/DataProcessTaskService.java
View File

@ -52,4 +52,14 @@ public interface DataProcessTaskService {
*/
PageResult<DataProcessTaskDO> getDataProcessTaskPage(DataProcessTaskPageReqVO pageReqVO);
void stopFineTuningTask(Long id);
void reStartFineTuningTask(Long id);
/**
* 分页查询数据处理任务
* @param pageReqVO
* @return
*/
PageResult<DataProcessTaskRespVO> getDataProcessTaskPage1(DataProcessTaskPageReqVO pageReqVO);
}

60
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataprocesstask/DataProcessTaskServiceImpl.java
View File

@ -1,11 +1,21 @@
package cn.iocoder.yudao.module.llm.service.dataprocesstask;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.service.async.AsyncDataProcessService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import org.springframework.validation.annotation.Validated;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
@ -28,16 +38,23 @@ public class DataProcessTaskServiceImpl implements DataProcessTaskService {
@Resource
private DataProcessTaskMapper dataProcessTaskMapper;
@Resource
private DatasetMapper datasetMapper;
@Resource
private AsyncDataProcessService dataProcessService;
@Override
public Long createDataProcessTask(DataProcessTaskSaveReqVO createReqVO) {
// 插入
DataProcessTaskDO dataProcessTask = BeanUtils.toBean(createReqVO, DataProcessTaskDO.class);
dataProcessTask.setStatus(1);
dataProcessTaskMapper.insert(dataProcessTask);
dataProcessService.backups(dataProcessTask);
// 返回
return dataProcessTask.getId();
}
@Override
public void updateDataProcessTask(DataProcessTaskSaveReqVO updateReqVO) {
// 校验存在
@ -70,5 +87,48 @@ public class DataProcessTaskServiceImpl implements DataProcessTaskService {
public PageResult<DataProcessTaskDO> getDataProcessTaskPage(DataProcessTaskPageReqVO pageReqVO) {
return dataProcessTaskMapper.selectPage(pageReqVO);
}
@Override
public PageResult<DataProcessTaskRespVO> getDataProcessTaskPage1(DataProcessTaskPageReqVO pageReqVO) {
PageResult<DataProcessTaskDO> dataProcessTaskDOPageResult = dataProcessTaskMapper.selectPage(pageReqVO);
PageResult<DataProcessTaskRespVO> result = BeanUtils.toBean(dataProcessTaskDOPageResult, DataProcessTaskRespVO.class);
if (!CollectionUtils.isAnyEmpty(dataProcessTaskDOPageResult.getList())){
List<Long> collect = dataProcessTaskDOPageResult.getList().stream()
.map(DataProcessTaskDO::getDatasetId)
.collect(Collectors.toList());
List<Long> collect1 = dataProcessTaskDOPageResult.getList()
.stream().map(DataProcessTaskDO::getDatasetPostId).collect(Collectors.toList());
Set<Long> datasetIds = Stream.concat(collect.stream(), collect1.stream()).collect(Collectors.toSet());
if (!CollectionUtils.isAnyEmpty(datasetIds)){
List<Long> mergeDataList = new ArrayList<>(datasetIds);
List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().in(DatasetDO::getId, mergeDataList));
Map<Long, DatasetDO> longDataDOMap = cn.iocoder.yudao.framework.common.util.collection.
CollectionUtils.convertMap(datasetDOS, DatasetDO::getId);
result.getList().forEach(item->{
DatasetDO datasetDO = longDataDOMap.get(item.getDatasetId());
if(datasetDO != null){
item.setDatasetName(datasetDO.getDatasetName());
}
if (item.getStatus() == 2){
DatasetDO datasetDO1 = longDataDOMap.get(item.getDatasetPostId());
if (datasetDO1 != null){
item.setDatasetPostName(datasetDO1.getDatasetName());
}
}
});
}
}
return result;
}
@Override
public void stopFineTuningTask(Long id) {
dataProcessTaskMapper.updateStatus(id, 3);
}
@Override
public void reStartFineTuningTask(Long id) {
dataProcessTaskMapper.updateStatus(id, 1);
}
}

7
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/datarefluxdata/DataRefluxDataService.java
View File

@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.llm.service.datarefluxdata;
import java.util.*;
import javax.validation.*;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxconfig.vo.DataRefluxConfigPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.datarefluxdata.DataRefluxDataDO;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
@ -52,4 +54,7 @@ public interface DataRefluxDataService {
*/
PageResult<DataRefluxDataDO> getDataRefluxDataPage(DataRefluxDataPageReqVO pageReqVO);
}
List<DataRefluxDataDO> getTheDetails(DataRefluxConfigPageReqVO pageReqVO);
Long saveDataRefluxData(DataRefluxDataSaveReqVO createReqVO);
}

43
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/datarefluxdata/DataRefluxDataServiceImpl.java
View File

@ -1,15 +1,18 @@
package cn.iocoder.yudao.module.llm.service.datarefluxdata;
import cn.hutool.core.util.ObjectUtil;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxconfig.vo.DataRefluxConfigPageReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.datarefluxconfig.DataRefluxConfigDO;
import cn.iocoder.yudao.module.llm.dal.mysql.datarefluxconfig.DataRefluxConfigMapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import org.springframework.validation.annotation.Validated;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.datarefluxdata.DataRefluxDataDO;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.dal.mysql.datarefluxdata.DataRefluxDataMapper;
@ -28,11 +31,19 @@ public class DataRefluxDataServiceImpl implements DataRefluxDataService {
@Resource
private DataRefluxDataMapper dataRefluxDataMapper;
@Resource
private DataRefluxConfigMapper dataRefluxConfigMapper;
@Override
public Long createDataRefluxData(DataRefluxDataSaveReqVO createReqVO) {
// 插入
DataRefluxDataDO dataRefluxData = BeanUtils.toBean(createReqVO, DataRefluxDataDO.class);
LambdaQueryWrapper<DataRefluxConfigDO> eq = new LambdaQueryWrapper<DataRefluxConfigDO>()
.eq(DataRefluxConfigDO::getModelServiceId, dataRefluxData.getModelServiceId())
.eq(DataRefluxConfigDO::getModelType, dataRefluxData.getModelType())
.like(DataRefluxConfigDO::getCreator, dataRefluxData.getCreator());
List<DataRefluxConfigDO> dataRefluxConfigDOS = dataRefluxConfigMapper.selectList(eq);
dataRefluxData.setConfigId(dataRefluxConfigDOS.get(0).getId());
dataRefluxDataMapper.insert(dataRefluxData);
// 返回
return dataRefluxData.getId();
@ -71,4 +82,30 @@ public class DataRefluxDataServiceImpl implements DataRefluxDataService {
return dataRefluxDataMapper.selectPage(pageReqVO);
}
}
@Override
public List<DataRefluxDataDO> getTheDetails(DataRefluxConfigPageReqVO pageReqVO) {
LambdaQueryWrapper<DataRefluxDataDO> eq = new LambdaQueryWrapper<DataRefluxDataDO>()
.eq(ObjectUtil.isNotNull(pageReqVO.getId()),DataRefluxDataDO::getConfigId, pageReqVO.getId())
.le(ObjectUtil.isNotNull(pageReqVO.getStartTime()),DataRefluxDataDO::getCreateTime, pageReqVO.getStartTime())
.ge(ObjectUtil.isNotNull(pageReqVO.getEndTime()),DataRefluxDataDO::getCreateTime, pageReqVO.getEndTime());
return dataRefluxDataMapper.selectList(eq);
}
@Override
public Long saveDataRefluxData(DataRefluxDataSaveReqVO createReqVO) {
// 插入
DataRefluxDataDO dataRefluxData = BeanUtils.toBean(createReqVO, DataRefluxDataDO.class);
LambdaQueryWrapper<DataRefluxConfigDO> eq = new LambdaQueryWrapper<DataRefluxConfigDO>()
.eq(DataRefluxConfigDO::getModelServiceId, dataRefluxData.getModelServiceId())
.eq(DataRefluxConfigDO::getModelType, dataRefluxData.getModelType())
.like(DataRefluxConfigDO::getCreator, dataRefluxData.getCreator());
List<DataRefluxConfigDO> dataRefluxConfigDOS = dataRefluxConfigMapper.selectList(eq);
if (!dataRefluxConfigDOS.isEmpty()) {
dataRefluxData.setConfigId(dataRefluxConfigDOS.get(0).getId());
dataRefluxDataMapper.insert(dataRefluxData);
// 返回
return dataRefluxData.getId();
}
return 0L;
}
}

55
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/DialogueHttpService.java Normal file
View File

@ -0,0 +1,55 @@
package cn.iocoder.yudao.module.llm.service.http;
import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Map;
/**
* 模型对话相关接口
*/
@Service
public class DialogueHttpService {
@Resource
private LLMBackendProperties llmBackendProperties;
/**
* 登录 POST
*/
public String login(Map<String, String> headers, String body){
String login = llmBackendProperties.getLogin();
String res = HttpUtils.post(login, headers, body);
return res;
}
/**
* 大模型列表 GET
*/
public String modelsList(Map<String, String> headers){
String modelsList = llmBackendProperties.getModelsList();
String res = HttpUtils.get(modelsList, headers);
return res;
}
/**
* 基础模型列表 GET
*/
public String baseModelList(Map<String, String> headers){
String baseModelList = llmBackendProperties.getBaseModelList();
String res = HttpUtils.get(baseModelList, headers);
return res;
}
/**
* 模型对话 POST
*/
public String modelCompletions(Map<String, String> headers, String body){
String finetuningCreate = llmBackendProperties.getModelCompletions();
String res = HttpUtils.post(finetuningCreate, headers, body);
return res;
}
}

4
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java
View File

@ -13,6 +13,9 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Map;
/**
* rag相关接口
*/
@Service
public class RagHttpService {
@ -83,5 +86,4 @@ public class RagHttpService {
return res;
}
}

124
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/TrainHttpService.java Normal file
View File

@ -0,0 +1,124 @@
package cn.iocoder.yudao.module.llm.service.http;
import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.vo.RagEmbedReqVo;
import cn.iocoder.yudao.module.llm.service.http.vo.RagQueryMultipleReqVo;
import cn.iocoder.yudao.module.llm.service.http.vo.RagQueryReqVo;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.toolkit.BeanUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Map;
/**
* 训练相关接口
*/
@Service
public class TrainHttpService {
@Resource
private LLMBackendProperties llmBackendProperties;
/**
* 训练集列表 GET
*/
public String datasetList(Map<String, String> headers){
String datasetList = llmBackendProperties.getDatasetList();
String res = HttpUtils.get(datasetList, headers);
return res;
}
/**
* 上传训练集 POST
*/
public String datasetCreate(Map<String, String> headers, String body){
String datasetCreate = llmBackendProperties.getDatasetCreate();
String res = HttpUtils.post(datasetCreate, headers, body);
return res;
}
/**
* 删除训练集 DELETE
*/
public String datasetDelete(Map<String,String> headers){
String datasetDelete = llmBackendProperties.getDatasetDelete();
String res = HttpUtils.del(datasetDelete, headers);
return res;
}
/**
* 训练集标注 GET
*/
public String annotationTaskList(Map<String, String> headers){
String annotationTaskList = llmBackendProperties.getAnnotationTaskList();
String res = HttpUtils.get(annotationTaskList, headers);
return res;
}
/**
* 标注信息 GET
*/
public String annotationTask(Map<String, String> headers){
String annotationTask = llmBackendProperties.getAnnotationTask();
String res = HttpUtils.get(annotationTask, headers);
return res;
}
/**
* 保存标注 POST
*/
public String annotationTaskSave(Map<String, String> headers, String body){
String annotationTaskSave = llmBackendProperties.getAnnotationTaskSave();
String res = HttpUtils.post(annotationTaskSave, headers, body);
return res;
}
/**
* 大模型列表 GET
*/
public String modelsList(Map<String, String> headers){
String modelsList = llmBackendProperties.getModelsList();
String res = HttpUtils.get(modelsList, headers);
return res;
}
/**
* 基础模型列表 GET
*/
public String baseModelList(Map<String, String> headers){
String baseModelList = llmBackendProperties.getBaseModelList();
String res = HttpUtils.get(baseModelList, headers);
return res;
}
/**
* 登录 POST
*/
public String login(Map<String, String> headers, String body){
String login = llmBackendProperties.getLogin();
String res = HttpUtils.post(login, headers, body);
return res;
}
/**
* 创建微调任务 POST
*/
public String finetuningCreate(Map<String, String> headers, String body){
String finetuningCreate = llmBackendProperties.getFinetuningCreate();
String res = HttpUtils.post(finetuningCreate, headers, body);
return res;
}
/**
* 模型对话 POST
*/
public String modelCompletions(Map<String, String> headers, String body){
String finetuningCreate = llmBackendProperties.getModelCompletions();
String res = HttpUtils.post(finetuningCreate, headers, body);
return res;
}
}

21
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/controller/admin/permission/RoleController.java
View File

@ -9,6 +9,7 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.excel.core.util.ExcelUtils;
import cn.iocoder.yudao.module.system.controller.admin.permission.vo.role.*;
import cn.iocoder.yudao.module.system.dal.dataobject.permission.RoleDO;
import cn.iocoder.yudao.module.system.service.permission.PermissionService;
import cn.iocoder.yudao.module.system.service.permission.RoleService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@ -23,6 +24,7 @@ import javax.validation.Valid;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import static cn.iocoder.yudao.framework.apilog.core.enums.OperateTypeEnum.EXPORT;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
@ -37,18 +39,28 @@ public class RoleController {
@Resource
private RoleService roleService;
@Resource
private PermissionService permissionService;
@PostMapping("/create")
@Operation(summary = "创建角色")
@PreAuthorize("@ss.hasPermission('system:role:create')")
public CommonResult<Long> createRole(@Valid @RequestBody RoleSaveReqVO createReqVO) {
return success(roleService.createRole(createReqVO, null));
//创建角色
Long roleId = roleService.createRole(createReqVO, null);
//赋予角色菜单权限
permissionService.assignRoleMenu(roleId, createReqVO.getMenuIds());
return success(roleId);
}
@PutMapping("/update")
@Operation(summary = "修改角色")
@PreAuthorize("@ss.hasPermission('system:role:update')")
public CommonResult<Boolean> updateRole(@Valid @RequestBody RoleSaveReqVO updateReqVO) {
//修改角色
roleService.updateRole(updateReqVO);
//修改角色菜单权限
permissionService.assignRoleMenu(updateReqVO.getId(), updateReqVO.getMenuIds());
return success(true);
}
@ -65,8 +77,13 @@ public class RoleController {
@Operation(summary = "获得角色信息")
@PreAuthorize("@ss.hasPermission('system:role:query')")
public CommonResult<RoleRespVO> getRole(@RequestParam("id") Long id) {
//获取角色信息
RoleDO role = roleService.getRole(id);
return success(BeanUtils.toBean(role, RoleRespVO.class));
//获取角色的菜单权限
Set<Long> roleMenuListByRoleId = permissionService.getRoleMenuListByRoleId(id);
RoleRespVO roleRespVO = BeanUtils.toBean(role, RoleRespVO.class);
roleRespVO.setMenuIds(roleMenuListByRoleId);
return success(roleRespVO);
}
@GetMapping("/page")

4
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/controller/admin/permission/vo/role/RoleRespVO.java
View File

@ -10,6 +10,7 @@ import lombok.Data;
import javax.validation.constraints.NotBlank;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Set;
@Schema(description = "管理后台 - 角色信息 Response VO")
@ -56,4 +57,7 @@ public class RoleRespVO {
@Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED, example = "时间戳格式")
private LocalDateTime createTime;
@Schema(description = "菜单编号列表", example = "1,3,5")
private Set<Long> menuIds = Collections.emptySet();
}

5
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/controller/admin/permission/vo/role/RoleSaveReqVO.java
View File

@ -9,6 +9,8 @@ import lombok.Data;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import java.util.Collections;
import java.util.Set;
@Schema(description = "管理后台 - 角色创建/更新 Request VO")
@Data
@ -45,4 +47,7 @@ public class RoleSaveReqVO {
@DiffLogField(name = "备注")
private String remark;
@Schema(description = "菜单编号列表", example = "1,3,5")
private Set<Long> menuIds = Collections.emptySet();
}

1
yudao-server/src/main/java/cn/iocoder/yudao/server/YudaoServerApplication.java
View File

@ -2,6 +2,7 @@ package cn.iocoder.yudao.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
/**
* 项目的启动类

8
yudao-server/src/main/resources/application-dev.yaml
View File

@ -236,10 +236,12 @@ llm:
models_list: http://localhost:8123/api/models
# 登录 POST
login: http://localhost:8123/login
# 创建微调任务 POST
finetuning_create: http://localhost:8123/api/finetuning
#### 大模型对话
# 模型列表 GET
model_list: http://api.xhllm.xinnuojinzhi.com/model/v1/models
# 基础模型列表 GET
base_model_list: http://api.xhllm.xinnuojinzhi.com/model/v1/models
# 模型对话 POST
model_completions: http://api.xhllm.xinnuojinzhi.com/model/v1/chat/completions
@ -259,4 +261,4 @@ iot:
# 保持连接
keepalive: 60
# 清除会话(设置为false,断开连接，重连后使用原来的会话 保留订阅的主题，能接收离线期间的消息)
clearSession: true
clearSession: true