From 2c55d4b9fe490296c9ba5dd35b5134d2d7a9fc50 Mon Sep 17 00:00:00 2001 From: limin Date: Tue, 31 Dec 2024 18:44:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=20=E5=BC=82=E6=AD=A5=20=E5=A2=9E=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataprocesstask/DataProcessTaskDO.java | 3 +- .../mysql/dataset/DatasetAnswerMapper.java | 5 +- .../llm/dal/mysql/dataset/DatasetMapper.java | 3 + .../mysql/dataset/DatasetQuestionMapper.java | 5 +- .../async/AsyncDataProcessService.java | 65 ++++++++++++++++++- .../dataset/DatasetQuestionService.java | 2 + .../dataset/DatasetQuestionServiceImpl.java | 14 ++++ 7 files changed, 91 insertions(+), 6 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataprocesstask/DataProcessTaskDO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataprocesstask/DataProcessTaskDO.java index 85a93938a..7b4274549 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataprocesstask/DataProcessTaskDO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataprocesstask/DataProcessTaskDO.java @@ -1,5 +1,6 @@ package cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask; +import cn.hutool.json.JSONObject; import lombok.*; import java.util.*; import java.time.LocalDateTime; @@ -46,6 +47,6 @@ public class DataProcessTaskDO extends BaseDO { /** * 配置信息 */ - private String options; + private JSONObject options; } \ No newline at end of file diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java index f0d4b8215..d2b402cbf 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java @@ -7,7 +7,9 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerPageReqVO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; /** * 数据集数据问题标注内容 Mapper @@ -26,5 +28,6 @@ public interface DatasetAnswerMapper extends BaseMapperX { .betweenIfPresent(DatasetAnswerDO::getCreateTime, reqVO.getCreateTime()) .orderByDesc(DatasetAnswerDO::getId)); } - + @Delete("delete from llm_dataset_answer where dataset_id = #{datasetPostId}") + void deleteTrue(@Param("datasetPostId") Long datasetPostId); } \ No newline at end of file diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java index 40ee082b4..b49a4ffa3 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java @@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*; import org.apache.ibatis.annotations.Param; @@ -35,4 +36,6 @@ public interface DatasetMapper extends BaseMapperX { @Update("update llm_dataset set annotate_progress = #{formattedRatio} where id = #{datasetId}") void updateProcess(@Param("formattedRatio") Integer formattedRatio, @Param("datasetId") Long datasetId); + @Delete("delete from llm_dataset where id = #{datasetPostId}") + void deleteTrue(@Param("datasetPostId") Long datasetPostId); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java index 0a83ec10a..f633a3d80 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java @@ -8,7 +8,9 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionPageReqVO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; /** @@ -28,5 +30,6 @@ public interface DatasetQuestionMapper extends BaseMapperX { .betweenIfPresent(DatasetQuestionDO::getCreateTime, reqVO.getCreateTime()) .orderByDesc(DatasetQuestionDO::getId)); } - + @Delete("delete from llm_dataset_question where dataset_id = #{datasetPostId}") + void deleteTrue(@Param("datasetPostId") Long datasetPostId); } \ No newline at end of file diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java index 6f61e6ae4..5adb95524 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java @@ -1,14 +1,23 @@ package cn.iocoder.yudao.module.llm.service.async; +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataprocesstask.DataProcessTaskDO; +import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO; +import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO; import cn.iocoder.yudao.module.llm.dal.mysql.dataprocesstask.DataProcessTaskMapper; +import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetAnswerMapper; import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper; +import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper; +import cn.iocoder.yudao.module.llm.service.dataset.DatasetQuestionService; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.security.SecureRandom; +import java.util.List; @Service public class AsyncDataProcessService { @@ -16,7 +25,14 @@ public class AsyncDataProcessService { private DatasetMapper datasetMapper; @Resource private DataProcessTaskMapper dataProcessTaskMapper; - + @Resource + private DatasetQuestionService datasetQuestionService; + @Resource + private DatasetQuestionMapper datasetQuestionMapper; + @Resource + private DatasetAnswerMapper datasetAnswerMapper; + private static final String CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + private static final SecureRandom RANDOM = new SecureRandom(); /* @Resource private */ @Async @@ -27,14 +43,57 @@ public class AsyncDataProcessService { DatasetDO datasetDO = datasetMapper.selectById(dataProcessTask.getDatasetId()); DatasetDO newData = BeanUtils.toBean(datasetDO, DatasetDO.class); newData.setId(null); + newData.setDatasetName(datasetDO.getDatasetName() + generateRandomString(4)); datasetMapper.insert(newData); dataProcessTask.setDatasetPostId(newData.getId()); dataProcessTaskMapper.updateById(dataProcessTask); } + // 删除备份后得副表数据 问题表 和答案表 + datasetQuestionMapper.deleteTrue(dataProcessTask.getDatasetPostId()); + datasetAnswerMapper.deleteTrue(dataProcessTask.getDatasetPostId()); - + List datasetQuestionList = datasetQuestionService.getDatasetQuestionList(dataProcessTask.getDatasetId()); + if (!CollectionUtils.isAnyEmpty(datasetQuestionList)){ + datasetQuestionList.forEach(item -> { + DatasetQuestionDO questionDO = BeanUtils.toBean(item, DatasetQuestionDO.class); + questionDO.setDatasetId(dataProcessTask.getDatasetPostId()); + questionDO.setId(null); + questionDO.setCreator(dataProcessTask.getCreator()); + questionDO.setUpdater(dataProcessTask.getUpdater()); + datasetQuestionMapper.insert(questionDO); + System.out.println(item); + System.out.println(questionDO); + if (!CollectionUtils.isAnyEmpty(item.getDatasetAnswerRespVO())){ + item.getDatasetAnswerRespVO().forEach(item1 -> { + DatasetAnswerDO answerDO = BeanUtils.toBean(item1, DatasetAnswerDO.class); + answerDO.setQuestionId(questionDO.getId()); + answerDO.setCreator(dataProcessTask.getCreator()); + answerDO.setUpdater(dataProcessTask.getCreator()); + answerDO.setId(null); + answerDO.setDatasetId(dataProcessTask.getDatasetPostId()); + datasetAnswerMapper.insert(answerDO); + dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 2); + }); + } + }); + } }catch(Exception e){ - dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 5); + datasetQuestionMapper.deleteTrue(dataProcessTask.getDatasetPostId()); + datasetAnswerMapper.deleteTrue(dataProcessTask.getDatasetPostId()); + datasetMapper.deleteTrue(dataProcessTask.getDatasetPostId()); + dataProcessTask.setDatasetPostId(null); + dataProcessTask.setStatus(5); + dataProcessTaskMapper.updateById(dataProcessTask); + +// dataProcessTaskMapper.updateStatus(dataProcessTask.getId(), 5); }; } + public static String generateRandomString(int length) { + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + int index = RANDOM.nextInt(CHARACTERS.length()); + sb.append(CHARACTERS.charAt(index)); + } + return sb.toString(); + } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionService.java index b74ba6c4a..54b7bb5fe 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionService.java @@ -55,4 +55,6 @@ public interface DatasetQuestionService { PageResult getDatasetQuestionPage(DatasetQuestionPageReqVO pageReqVO); void updateDatasetQuestionDataAnno(List updateReqVOS); + + List getDatasetQuestionList(Long datasetId); } \ No newline at end of file diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java index 21555a0a2..5e8bb246f 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java @@ -91,6 +91,20 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { } return result; } + @Override + public List getDatasetQuestionList(Long datasetId) { + List datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class) + .eq(DatasetQuestionDO::getDatasetId, datasetId)); + List result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class); + if (CollectionUtils.isNotEmpty(result)) { + result.forEach(item -> { + List datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class) + .eq(DatasetAnswerDO::getQuestionId, item.getId())); + item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class)); + }); + } + return result; + } @Override public void updateDatasetQuestionDataAnno(List updateReqVOS) {