DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

数据处理任务 异步 增加

Browse Source
This commit is contained in:
limin 2024-12-31 18:44:39 +08:00
parent 163bdef8a4
commit 2c55d4b9fe
7 changed files with 91 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataprocesstask/DataProcessTaskDO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask;
import cn.hutool.json.JSONObject;
import lombok.*;
import java.util.*;
import java.time.LocalDateTime;
@ -46,6 +47,6 @@ public class DataProcessTaskDO extends BaseDO {
/**
* 配置信息
*/
private String options;
private JSONObject options;
}

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java
View File

@ -7,7 +7,9 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerPageReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
* 数据集数据问题标注内容 Mapper
@ -26,5 +28,6 @@ public interface DatasetAnswerMapper extends BaseMapperX<DatasetAnswerDO> {
.betweenIfPresent(DatasetAnswerDO::getCreateTime, reqVO.getCreateTime())
.orderByDesc(DatasetAnswerDO::getId));
}
@Delete("delete from llm_dataset_answer where dataset_id = #{datasetPostId}")
void deleteTrue(@Param("datasetPostId") Long datasetPostId);
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java
View File

@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
import org.apache.ibatis.annotations.Param;
@ -35,4 +36,6 @@ public interface DatasetMapper extends BaseMapperX<DatasetDO> {
@Update("update llm_dataset set annotate_progress = #{formattedRatio} where id = #{datasetId}")
void updateProcess(@Param("formattedRatio") Integer formattedRatio, @Param("datasetId") Long datasetId);
@Delete("delete from llm_dataset where id = #{datasetPostId}")
void deleteTrue(@Param("datasetPostId") Long datasetPostId);
}

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java
View File

@ -8,7 +8,9 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionPageReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
@ -28,5 +30,6 @@ public interface DatasetQuestionMapper extends BaseMapperX<DatasetQuestionDO> {
.betweenIfPresent(DatasetQuestionDO::getCreateTime, reqVO.getCreateTime())
.orderByDesc(DatasetQuestionDO::getId));
}
@Delete("delete from llm_dataset_question where dataset_id = #{datasetPostId}")
void deleteTrue(@Param("datasetPostId") Long datasetPostId);
}

65
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java
View File

@ -1,14 +1,23 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataprocesstask.DataProcessTaskMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetAnswerMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper;
import cn.iocoder.yudao.module.llm.service.dataset.DatasetQuestionService;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.security.SecureRandom;
import java.util.List;
@Service
public class AsyncDataProcessService {
@ -16,7 +25,14 @@ public class AsyncDataProcessService {
private DatasetMapper datasetMapper;
@Resource
private DataProcessTaskMapper dataProcessTaskMapper;
@Resource
private DatasetQuestionService datasetQuestionService;
@Resource
private DatasetQuestionMapper datasetQuestionMapper;
@Resource
private DatasetAnswerMapper datasetAnswerMapper;
private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
private static final SecureRandom RANDOM = new SecureRandom();
/* @Resource
private */
@Async
@ -27,14 +43,57 @@ public class AsyncDataProcessService {
DatasetDO datasetDO = datasetMapper.selectById(dataProcessTask.getDatasetId());
DatasetDO newData = BeanUtils.toBean(datasetDO, DatasetDO.class);
newData.setId(null);
newData.setDatasetName(datasetDO.getDatasetName() + generateRandomString(4));
datasetMapper.insert(newData);
dataProcessTask.setDatasetPostId(newData.getId());
dataProcessTaskMapper.updateById(dataProcessTask);
}
// 删除备份后得副表数据 问题表 和答案表
datasetQuestionMapper.deleteTrue(dataProcessTask.getDatasetPostId());
datasetAnswerMapper.deleteTrue(dataProcessTask.getDatasetPostId());
List<DatasetQuestionRespVO> datasetQuestionList = datasetQuestionService.getDatasetQuestionList(dataProcessTask.getDatasetId());
if (!CollectionUtils.isAnyEmpty(datasetQuestionList)){
datasetQuestionList.forEach(item -> {
DatasetQuestionDO questionDO = BeanUtils.toBean(item, DatasetQuestionDO.class);
questionDO.setDatasetId(dataProcessTask.getDatasetPostId());
questionDO.setId(null);
questionDO.setCreator(dataProcessTask.getCreator());
questionDO.setUpdater(dataProcessTask.getUpdater());
datasetQuestionMapper.insert(questionDO);
System.out.println(item);
System.out.println(questionDO);
if (!CollectionUtils.isAnyEmpty(item.getDatasetAnswerRespVO())){
item.getDatasetAnswerRespVO().forEach(item1 -> {
DatasetAnswerDO answerDO = BeanUtils.toBean(item1, DatasetAnswerDO.class);
answerDO.setQuestionId(questionDO.getId());
answerDO.setCreator(dataProcessTask.getCreator());
answerDO.setUpdater(dataProcessTask.getCreator());
answerDO.setId(null);
answerDO.setDatasetId(dataProcessTask.getDatasetPostId());
datasetAnswerMapper.insert(answerDO);
dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 2);
});
}
});
}
}catch(Exception e){
dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 5);
datasetQuestionMapper.deleteTrue(dataProcessTask.getDatasetPostId());
datasetAnswerMapper.deleteTrue(dataProcessTask.getDatasetPostId());
datasetMapper.deleteTrue(dataProcessTask.getDatasetPostId());
dataProcessTask.setDatasetPostId(null);
dataProcessTask.setStatus(5);
dataProcessTaskMapper.updateById(dataProcessTask);
// dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 5);
};
}
public static String generateRandomString(int length) {
StringBuilder sb = new StringBuilder(length);
for (int i = 0; i < length; i++) {
int index = RANDOM.nextInt(CHARACTERS.length());
sb.append(CHARACTERS.charAt(index));
}
return sb.toString();
}
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionService.java
View File

@ -55,4 +55,6 @@ public interface DatasetQuestionService {
PageResult<DatasetQuestionRespVO> getDatasetQuestionPage(DatasetQuestionPageReqVO pageReqVO);
void updateDatasetQuestionDataAnno(List<DatasetQuestionSaveReqVO> updateReqVOS);
List<DatasetQuestionRespVO> getDatasetQuestionList(Long datasetId);
}

14
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java
View File

@ -91,6 +91,20 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
}
return result;
}
@Override
public List<DatasetQuestionRespVO> getDatasetQuestionList(Long datasetId) {
List<DatasetQuestionDO> datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class)
.eq(DatasetQuestionDO::getDatasetId, datasetId));
List<DatasetQuestionRespVO> result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class);
if (CollectionUtils.isNotEmpty(result)) {
result.forEach(item -> {
List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
.eq(DatasetAnswerDO::getQuestionId, item.getId()));
item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class));
});
}
return result;
}
@Override
public void updateDatasetQuestionDataAnno(List<DatasetQuestionSaveReqVO> updateReqVOS) {