DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

数据处理增加

Browse Source
This commit is contained in:
limin 2024-12-31 15:49:27 +08:00
parent 13554520be
commit 9ed36910f6
10 changed files with 139 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/DataProcessTaskController.java
View File

@ -75,8 +75,7 @@ public class DataProcessTaskController {
@Operation(summary = "获得数据处理任务分页")
@PreAuthorize("@ss.hasPermission('llm:data-process-task:query')")
public CommonResult<PageResult<DataProcessTaskRespVO>> getDataProcessTaskPage(@Valid DataProcessTaskPageReqVO pageReqVO) {
PageResult<DataProcessTaskDO> pageResult = dataProcessTaskService.getDataProcessTaskPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, DataProcessTaskRespVO.class));
return success(dataProcessTaskService.getDataProcessTaskPage1(pageReqVO));
}
@GetMapping("/export-excel")
@ -92,4 +91,19 @@ public class DataProcessTaskController {
BeanUtils.toBean(list, DataProcessTaskRespVO.class));
}
@PutMapping("/stop")
@Operation(summary = "停止模型调优 —— 微调任务")
@PreAuthorize("@ss.hasPermission('llm:fine-tuning-task:update')")
public CommonResult<Boolean> stopFineTuningTask(@RequestParam("id") Long id) {
dataProcessTaskService.stopFineTuningTask(id);
return success(true);
}
@PutMapping("/reStart")
@Operation(summary = "开始模型调优 —— 微调任务")
@PreAuthorize("@ss.hasPermission('llm:fine-tuning-task:update')")
public CommonResult<Boolean> startFineTuningTask(@RequestParam("id") Long id) {
dataProcessTaskService.reStartFineTuningTask(id);
return success(true);
}
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskPageReqVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import lombok.*;
import java.util.*;
import io.swagger.v3.oas.annotations.media.Schema;
@ -32,6 +33,6 @@ public class DataProcessTaskPageReqVO extends PageParam {
private Long datasetPostId;
@Schema(description = "配置信息")
private String options;
private JSONObject options;
}

9
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskRespVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -38,6 +39,12 @@ public class DataProcessTaskRespVO {
@Schema(description = "配置信息")
@ExcelProperty("配置信息")
private String options;
private JSONObject options;
@Schema(description ="处理前的数据集名称")
private String datasetName;
@Schema(description = "处理后的数据集名称")
private String datasetPostName;
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataprocesstask/vo/DataProcessTaskSaveReqVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo;
import cn.hutool.json.JSONObject;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
@ -27,6 +28,6 @@ public class DataProcessTaskSaveReqVO {
private Long datasetPostId;
@Schema(description = "配置信息")
private String options;
private JSONObject options;
}

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataprocesstask/DataProcessTaskMapper.java
View File

@ -8,6 +8,8 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import org.apache.ibatis.annotations.Mapper;
import cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo.*;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
/**
* 数据处理任务 Mapper
@ -27,5 +29,6 @@ public interface DataProcessTaskMapper extends BaseMapperX<DataProcessTaskDO> {
.eqIfPresent(DataProcessTaskDO::getOptions, reqVO.getOptions())
.orderByDesc(DataProcessTaskDO::getId));
}
@Update("update llm_data_process_task set status = #{status} where id = #{id}")
void updateStatus(@Param("id") Long id, @Param("status") int status);
}

16
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/AsyncConfig.java Normal file
View File

@ -0,0 +1,16 @@
package cn.iocoder.yudao.module.llm.framework.backend.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import java.util.concurrent.Executor;
@Configuration
@EnableAsync
public class AsyncConfig implements AsyncConfigurer {
@Override
public Executor getAsyncExecutor() {
return AsyncConfigurer.super.getAsyncExecutor();
}
}

20
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java Normal file
View File

@ -0,0 +1,20 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@Service
public class AsyncDataProcessService {
@Async
public void backups(DataProcessTaskDO dataProcessTask) {
try {
Thread.sleep(100000);
}catch(InterruptedException e){
};
}
}

10
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataprocesstask/DataProcessTaskService.java
View File

@ -52,4 +52,14 @@ public interface DataProcessTaskService {
*/
PageResult<DataProcessTaskDO> getDataProcessTaskPage(DataProcessTaskPageReqVO pageReqVO);
void stopFineTuningTask(Long id);
void reStartFineTuningTask(Long id);
/**
* 分页查询数据处理任务
* @param pageReqVO
* @return
*/
PageResult<DataProcessTaskRespVO> getDataProcessTaskPage1(DataProcessTaskPageReqVO pageReqVO);
}

60
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataprocesstask/DataProcessTaskServiceImpl.java
View File

@ -1,11 +1,21 @@
package cn.iocoder.yudao.module.llm.service.dataprocesstask;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.service.async.AsyncDataProcessService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import org.springframework.validation.annotation.Validated;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import cn.iocoder.yudao.module.llm.controller.admin.dataprocesstask.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
@ -28,16 +38,23 @@ public class DataProcessTaskServiceImpl implements DataProcessTaskService {
@Resource
private DataProcessTaskMapper dataProcessTaskMapper;
@Resource
private DatasetMapper datasetMapper;
@Resource
private AsyncDataProcessService dataProcessService;
@Override
public Long createDataProcessTask(DataProcessTaskSaveReqVO createReqVO) {
// 插入
DataProcessTaskDO dataProcessTask = BeanUtils.toBean(createReqVO, DataProcessTaskDO.class);
dataProcessTask.setStatus(1);
dataProcessTaskMapper.insert(dataProcessTask);
dataProcessService.backups(dataProcessTask);
// 返回
return dataProcessTask.getId();
}
@Override
public void updateDataProcessTask(DataProcessTaskSaveReqVO updateReqVO) {
// 校验存在
@ -70,5 +87,48 @@ public class DataProcessTaskServiceImpl implements DataProcessTaskService {
public PageResult<DataProcessTaskDO> getDataProcessTaskPage(DataProcessTaskPageReqVO pageReqVO) {
return dataProcessTaskMapper.selectPage(pageReqVO);
}
@Override
public PageResult<DataProcessTaskRespVO> getDataProcessTaskPage1(DataProcessTaskPageReqVO pageReqVO) {
PageResult<DataProcessTaskDO> dataProcessTaskDOPageResult = dataProcessTaskMapper.selectPage(pageReqVO);
PageResult<DataProcessTaskRespVO> result = BeanUtils.toBean(dataProcessTaskDOPageResult, DataProcessTaskRespVO.class);
if (!CollectionUtils.isAnyEmpty(dataProcessTaskDOPageResult.getList())){
List<Long> collect = dataProcessTaskDOPageResult.getList().stream()
.map(DataProcessTaskDO::getDatasetId)
.collect(Collectors.toList());
List<Long> collect1 = dataProcessTaskDOPageResult.getList()
.stream().map(DataProcessTaskDO::getDatasetPostId).collect(Collectors.toList());
Set<Long> datasetIds = Stream.concat(collect.stream(), collect1.stream()).collect(Collectors.toSet());
if (!CollectionUtils.isAnyEmpty(datasetIds)){
List<Long> mergeDataList = new ArrayList<>(datasetIds);
List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().in(DatasetDO::getId, mergeDataList));
Map<Long, DatasetDO> longDataDOMap = cn.iocoder.yudao.framework.common.util.collection.
CollectionUtils.convertMap(datasetDOS, DatasetDO::getId);
result.getList().forEach(item->{
DatasetDO datasetDO = longDataDOMap.get(item.getDatasetId());
if(datasetDO != null){
item.setDatasetName(datasetDO.getDatasetName());
}
if (item.getStatus() == 2){
DatasetDO datasetDO1 = longDataDOMap.get(item.getDatasetPostId());
if (datasetDO1 != null){
item.setDatasetPostName(datasetDO1.getDatasetName());
}
}
});
}
}
return result;
}
@Override
public void stopFineTuningTask(Long id) {
dataProcessTaskMapper.updateStatus(id, 3);
}
@Override
public void reStartFineTuningTask(Long id) {
dataProcessTaskMapper.updateStatus(id, 1);
}
}

1
yudao-server/src/main/java/cn/iocoder/yudao/server/YudaoServerApplication.java
View File

@ -2,6 +2,7 @@ package cn.iocoder.yudao.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
/**
* 项目的启动类