diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java index d2b402cbf..63b0e4d3f 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetAnswerMapper.java @@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerPageReqVO; +import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerRespVO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; @@ -30,4 +31,6 @@ public interface DatasetAnswerMapper extends BaseMapperX { } @Delete("delete from llm_dataset_answer where dataset_id = #{datasetPostId}") void deleteTrue(@Param("datasetPostId") Long datasetPostId); -} \ No newline at end of file + + List getAnswersToYourQuestions(@Param("collected") List collected); +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java index f633a3d80..f47b62cf2 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionMapper.java @@ -7,6 +7,7 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionPageReqVO; +import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO; import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; @@ -32,4 +33,6 @@ public interface DatasetQuestionMapper extends BaseMapperX { } @Delete("delete from llm_dataset_question where dataset_id = #{datasetPostId}") void deleteTrue(@Param("datasetPostId") Long datasetPostId); -} \ No newline at end of file + + List getAListOfIssues(@Param("datasetId") Long datasetId); +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java index 98e61c43a..7c2f954c1 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java @@ -10,7 +10,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import jodd.util.StringUtil; import org.springframework.stereotype.Service; + import javax.annotation.Resource; + import org.springframework.validation.annotation.Validated; import org.springframework.transaction.annotation.Transactional; @@ -82,7 +84,7 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { public PageResult getDatasetQuestionPage(DatasetQuestionPageReqVO pageReqVO) { PageResult datasetQuestionDOPageResult = datasetQuestionMapper.selectPage(pageReqVO); PageResult result = BeanUtils.toBean(datasetQuestionDOPageResult, DatasetQuestionRespVO.class); - if (CollectionUtils.isNotEmpty(result.getList())){ + if (CollectionUtils.isNotEmpty(result.getList())) { result.getList().forEach(item -> { List datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class) .eq(DatasetAnswerDO::getQuestionId, item.getId())); @@ -101,15 +103,42 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { */ @Override public List getDatasetQuestionList(Long datasetId) { - List datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class) - .eq(DatasetQuestionDO::getDatasetId, datasetId)); - List result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class); +// List datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class) +// .eq(DatasetQuestionDO::getDatasetId, datasetId)); +// List result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class); + List result = datasetQuestionMapper.getAListOfIssues(datasetId); + if (CollectionUtils.isNotEmpty(result)) { + // 1. 获取待查询的 questionId 列表(假设 result 已分页,避免过大) + List collected = result.stream() + .map(DatasetQuestionRespVO::getId) + .collect(Collectors.toList()); + + // 2. 分批次查询答案(避免单次 IN 语句过长) + int batchSize = 1000; + List allAnswers = new ArrayList<>(); + for (int i = 0; i < collected.size(); i += batchSize) { + int end = Math.min(i + batchSize, collected.size()); + List batchIds = collected.subList(i, end); + allAnswers.addAll(datasetAnswerMapper.getAnswersToYourQuestions(batchIds)); + } + + // 3. 构建 questionId 到答案列表的映射(加速匹配) + Map> answerMap = new HashMap<>(); + for (DatasetAnswerRespVO answer : allAnswers) { + answerMap.computeIfAbsent(answer.getQuestionId(), k -> new ArrayList<>()) + .add(answer); + } + +// 4. 为每个 question 绑定答案(O(n) 效率) result.forEach(item -> { - List datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class) - .eq(DatasetAnswerDO::getQuestionId, item.getId())); - item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class)); + item.setDatasetAnswerRespVO(answerMap.getOrDefault(item.getId(), Collections.emptyList())); }); +// result.forEach(item -> { +// List datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class) +// .eq(DatasetAnswerDO::getQuestionId, item.getId())); +// item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class)); +// }); } return result; } @@ -118,23 +147,23 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { public void updateDatasetQuestionDataAnno(List updateReqVOS) { List ids = new ArrayList<>(); Long datasetId = null; - for (DatasetQuestionSaveReqVO updateReqVO : updateReqVOS){ - if (datasetId == null){ + for (DatasetQuestionSaveReqVO updateReqVO : updateReqVOS) { + if (datasetId == null) { datasetId = updateReqVO.getDatasetId(); } - DatasetQuestionDO datasetQuestionDO = BeanUtils.toBean(updateReqVO, DatasetQuestionDO.class); + DatasetQuestionDO datasetQuestionDO = BeanUtils.toBean(updateReqVO, DatasetQuestionDO.class); List datasetAnswerSaveReqVO = updateReqVO.getDatasetAnswerRespVO(); List datasetAnswerDOS = BeanUtils.toBean(datasetAnswerSaveReqVO, DatasetAnswerDO.class); - if (CollectionUtils.isNotEmpty(datasetAnswerDOS)){ - for (DatasetAnswerDO datasetAnswerDO : datasetAnswerDOS){ - if (StringUtil.isNotBlank(datasetAnswerDO.getAnswer())){ + if (CollectionUtils.isNotEmpty(datasetAnswerDOS)) { + for (DatasetAnswerDO datasetAnswerDO : datasetAnswerDOS) { + if (StringUtil.isNotBlank(datasetAnswerDO.getAnswer())) { datasetQuestionDO.setStatus(2); } - if (datasetAnswerDO.getId() == null){ + if (datasetAnswerDO.getId() == null) { datasetAnswerMapper.insert(datasetAnswerDO); ids.add(datasetAnswerDO.getId()); - }else { + } else { ids.add(datasetAnswerDO.getId()); datasetAnswerMapper.updateById(datasetAnswerDO); } @@ -145,30 +174,30 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { } List collect1 = updateReqVOS.stream().map(DatasetQuestionSaveReqVO::getId).collect(Collectors.toList()); LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.in(collect1!= null,DatasetAnswerDO::getQuestionId, collect1); + queryWrapper.in(collect1 != null, DatasetAnswerDO::getQuestionId, collect1); List datasetAnswerDOS = datasetAnswerMapper.selectList(queryWrapper); List collect = datasetAnswerDOS.stream().map(DatasetAnswerDO::getId).collect(Collectors.toList()); List diff1 = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(collect)){ + if (CollectionUtils.isNotEmpty(collect)) { HashSet set1 = new HashSet<>(ids); HashSet set2 = new HashSet<>(collect); // 获取 set2 中有但 set1 中没有的元素 set2.removeAll(set1); diff1 = new ArrayList<>(set2); } - if (CollectionUtils.isNotEmpty(diff1)){ + if (CollectionUtils.isNotEmpty(diff1)) { datasetAnswerMapper.deleteBatchIds(diff1); } // 标注进度修改 - LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() .eq(DatasetQuestionDO::getDatasetId, updateReqVOS.get(0).getDatasetId()); Long sumCount = datasetQuestionMapper.selectCount(wrapper); - wrapper.eq(DatasetQuestionDO::getStatus,2); + wrapper.eq(DatasetQuestionDO::getStatus, 2); Long annoCount = datasetQuestionMapper.selectCount(wrapper); - double ratio = sumCount == 0 ? 0 : ((double) annoCount / sumCount) *100; + double ratio = sumCount == 0 ? 0 : ((double) annoCount / sumCount) * 100; Integer formattedRatio = ratio == 0 ? 0 : (int) ratio; Integer status = formattedRatio == 100 ? 2 : 1; - datasetMapper.updateProcess(formattedRatio, updateReqVOS.get(0).getDatasetId(),status); + datasetMapper.updateProcess(formattedRatio, updateReqVOS.get(0).getDatasetId(), status); } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetAnswerMapper.xml b/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetAnswerMapper.xml index f098de38a..11c667c57 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetAnswerMapper.xml +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetAnswerMapper.xml @@ -9,4 +9,21 @@ 文档可见:https://www.iocoder.cn/MyBatis/x-plugins/ --> - \ No newline at end of file + + diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetQuestionMapper.xml b/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetQuestionMapper.xml index 9382711c2..7548f4a06 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetQuestionMapper.xml +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetQuestionMapper.xml @@ -9,4 +9,18 @@ 文档可见:https://www.iocoder.cn/MyBatis/x-plugins/ --> - \ No newline at end of file + +