feat(llm): 添加数据集相关查询功能
- 在 DatasetAnswerMapper 中添加 getAnswersToYourQuestions 方法，用于查询指定问题的解答- 在 DatasetQuestionMapper 中添加 getAListOfIssues 方法，用于查询指定数据集的问题列表 - 相应地在 XML 文件中添加了对应的 SQL 查询语句
This commit is contained in:
parent
627e031fcb
commit
6871b9d1dd
@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerPageReqVO;
|
||||
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerRespVO;
|
||||
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
@ -30,4 +31,6 @@ public interface DatasetAnswerMapper extends BaseMapperX<DatasetAnswerDO> {
|
||||
}
|
||||
@Delete("delete from llm_dataset_answer where dataset_id = #{datasetPostId}")
|
||||
void deleteTrue(@Param("datasetPostId") Long datasetPostId);
|
||||
}
|
||||
|
||||
List<DatasetAnswerRespVO> getAnswersToYourQuestions(@Param("collected") List<Long> collected);
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
|
||||
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionPageReqVO;
|
||||
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
|
||||
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
@ -32,4 +33,6 @@ public interface DatasetQuestionMapper extends BaseMapperX<DatasetQuestionDO> {
|
||||
}
|
||||
@Delete("delete from llm_dataset_question where dataset_id = #{datasetPostId}")
|
||||
void deleteTrue(@Param("datasetPostId") Long datasetPostId);
|
||||
}
|
||||
|
||||
List<DatasetQuestionRespVO> getAListOfIssues(@Param("datasetId") Long datasetId);
|
||||
}
|
||||
|
||||
@ -10,7 +10,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
|
||||
import jodd.util.StringUtil;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@ -82,7 +84,7 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
|
||||
public PageResult<DatasetQuestionRespVO> getDatasetQuestionPage(DatasetQuestionPageReqVO pageReqVO) {
|
||||
PageResult<DatasetQuestionDO> datasetQuestionDOPageResult = datasetQuestionMapper.selectPage(pageReqVO);
|
||||
PageResult<DatasetQuestionRespVO> result = BeanUtils.toBean(datasetQuestionDOPageResult, DatasetQuestionRespVO.class);
|
||||
if (CollectionUtils.isNotEmpty(result.getList())){
|
||||
if (CollectionUtils.isNotEmpty(result.getList())) {
|
||||
result.getList().forEach(item -> {
|
||||
List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
|
||||
.eq(DatasetAnswerDO::getQuestionId, item.getId()));
|
||||
@ -101,15 +103,42 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
|
||||
*/
|
||||
@Override
|
||||
public List<DatasetQuestionRespVO> getDatasetQuestionList(Long datasetId) {
|
||||
List<DatasetQuestionDO> datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class)
|
||||
.eq(DatasetQuestionDO::getDatasetId, datasetId));
|
||||
List<DatasetQuestionRespVO> result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class);
|
||||
// List<DatasetQuestionDO> datasetQuestionDOS = datasetQuestionMapper.selectList(new LambdaQueryWrapper<>(DatasetQuestionDO.class)
|
||||
// .eq(DatasetQuestionDO::getDatasetId, datasetId));
|
||||
// List<DatasetQuestionRespVO> result = BeanUtils.toBean(datasetQuestionDOS, DatasetQuestionRespVO.class);
|
||||
List<DatasetQuestionRespVO> result = datasetQuestionMapper.getAListOfIssues(datasetId);
|
||||
|
||||
if (CollectionUtils.isNotEmpty(result)) {
|
||||
// 1. 获取待查询的 questionId 列表（假设 result 已分页，避免过大）
|
||||
List<Long> collected = result.stream()
|
||||
.map(DatasetQuestionRespVO::getId)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// 2. 分批次查询答案（避免单次 IN 语句过长）
|
||||
int batchSize = 1000;
|
||||
List<DatasetAnswerRespVO> allAnswers = new ArrayList<>();
|
||||
for (int i = 0; i < collected.size(); i += batchSize) {
|
||||
int end = Math.min(i + batchSize, collected.size());
|
||||
List<Long> batchIds = collected.subList(i, end);
|
||||
allAnswers.addAll(datasetAnswerMapper.getAnswersToYourQuestions(batchIds));
|
||||
}
|
||||
|
||||
// 3. 构建 questionId 到答案列表的映射（加速匹配）
|
||||
Map<Long, List<DatasetAnswerRespVO>> answerMap = new HashMap<>();
|
||||
for (DatasetAnswerRespVO answer : allAnswers) {
|
||||
answerMap.computeIfAbsent(answer.getQuestionId(), k -> new ArrayList<>())
|
||||
.add(answer);
|
||||
}
|
||||
|
||||
// 4. 为每个 question 绑定答案（O(n) 效率）
|
||||
result.forEach(item -> {
|
||||
List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
|
||||
.eq(DatasetAnswerDO::getQuestionId, item.getId()));
|
||||
item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class));
|
||||
item.setDatasetAnswerRespVO(answerMap.getOrDefault(item.getId(), Collections.emptyList()));
|
||||
});
|
||||
// result.forEach(item -> {
|
||||
// List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
|
||||
// .eq(DatasetAnswerDO::getQuestionId, item.getId()));
|
||||
// item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class));
|
||||
// });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -118,23 +147,23 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
|
||||
public void updateDatasetQuestionDataAnno(List<DatasetQuestionSaveReqVO> updateReqVOS) {
|
||||
List<Long> ids = new ArrayList<>();
|
||||
Long datasetId = null;
|
||||
for (DatasetQuestionSaveReqVO updateReqVO : updateReqVOS){
|
||||
if (datasetId == null){
|
||||
for (DatasetQuestionSaveReqVO updateReqVO : updateReqVOS) {
|
||||
if (datasetId == null) {
|
||||
datasetId = updateReqVO.getDatasetId();
|
||||
}
|
||||
DatasetQuestionDO datasetQuestionDO = BeanUtils.toBean(updateReqVO, DatasetQuestionDO.class);
|
||||
DatasetQuestionDO datasetQuestionDO = BeanUtils.toBean(updateReqVO, DatasetQuestionDO.class);
|
||||
|
||||
List<DatasetAnswerSaveReqVO> datasetAnswerSaveReqVO = updateReqVO.getDatasetAnswerRespVO();
|
||||
List<DatasetAnswerDO> datasetAnswerDOS = BeanUtils.toBean(datasetAnswerSaveReqVO, DatasetAnswerDO.class);
|
||||
if (CollectionUtils.isNotEmpty(datasetAnswerDOS)){
|
||||
for (DatasetAnswerDO datasetAnswerDO : datasetAnswerDOS){
|
||||
if (StringUtil.isNotBlank(datasetAnswerDO.getAnswer())){
|
||||
if (CollectionUtils.isNotEmpty(datasetAnswerDOS)) {
|
||||
for (DatasetAnswerDO datasetAnswerDO : datasetAnswerDOS) {
|
||||
if (StringUtil.isNotBlank(datasetAnswerDO.getAnswer())) {
|
||||
datasetQuestionDO.setStatus(2);
|
||||
}
|
||||
if (datasetAnswerDO.getId() == null){
|
||||
if (datasetAnswerDO.getId() == null) {
|
||||
datasetAnswerMapper.insert(datasetAnswerDO);
|
||||
ids.add(datasetAnswerDO.getId());
|
||||
}else {
|
||||
} else {
|
||||
ids.add(datasetAnswerDO.getId());
|
||||
datasetAnswerMapper.updateById(datasetAnswerDO);
|
||||
}
|
||||
@ -145,30 +174,30 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
|
||||
}
|
||||
List<Long> collect1 = updateReqVOS.stream().map(DatasetQuestionSaveReqVO::getId).collect(Collectors.toList());
|
||||
LambdaQueryWrapper<DatasetAnswerDO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.in(collect1!= null,DatasetAnswerDO::getQuestionId, collect1);
|
||||
queryWrapper.in(collect1 != null, DatasetAnswerDO::getQuestionId, collect1);
|
||||
List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(queryWrapper);
|
||||
List<Long> collect = datasetAnswerDOS.stream().map(DatasetAnswerDO::getId).collect(Collectors.toList());
|
||||
List<Long> diff1 = new ArrayList<>();
|
||||
if (CollectionUtils.isNotEmpty(collect)){
|
||||
if (CollectionUtils.isNotEmpty(collect)) {
|
||||
HashSet<Long> set1 = new HashSet<>(ids);
|
||||
HashSet<Long> set2 = new HashSet<>(collect);
|
||||
// 获取 set2 中有但 set1 中没有的元素
|
||||
set2.removeAll(set1);
|
||||
diff1 = new ArrayList<>(set2);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(diff1)){
|
||||
if (CollectionUtils.isNotEmpty(diff1)) {
|
||||
datasetAnswerMapper.deleteBatchIds(diff1);
|
||||
}
|
||||
// 标注进度修改
|
||||
LambdaQueryWrapper<DatasetQuestionDO> wrapper = new LambdaQueryWrapper<DatasetQuestionDO>()
|
||||
LambdaQueryWrapper<DatasetQuestionDO> wrapper = new LambdaQueryWrapper<DatasetQuestionDO>()
|
||||
.eq(DatasetQuestionDO::getDatasetId, updateReqVOS.get(0).getDatasetId());
|
||||
Long sumCount = datasetQuestionMapper.selectCount(wrapper);
|
||||
wrapper.eq(DatasetQuestionDO::getStatus,2);
|
||||
wrapper.eq(DatasetQuestionDO::getStatus, 2);
|
||||
Long annoCount = datasetQuestionMapper.selectCount(wrapper);
|
||||
double ratio = sumCount == 0 ? 0 : ((double) annoCount / sumCount) *100;
|
||||
double ratio = sumCount == 0 ? 0 : ((double) annoCount / sumCount) * 100;
|
||||
Integer formattedRatio = ratio == 0 ? 0 : (int) ratio;
|
||||
Integer status = formattedRatio == 100 ? 2 : 1;
|
||||
datasetMapper.updateProcess(formattedRatio, updateReqVOS.get(0).getDatasetId(),status);
|
||||
datasetMapper.updateProcess(formattedRatio, updateReqVOS.get(0).getDatasetId(), status);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -9,4 +9,21 @@
|
||||
文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
|
||||
-->
|
||||
|
||||
</mapper>
|
||||
<select id="getAnswersToYourQuestions"
|
||||
resultType="cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerRespVO">
|
||||
SELECT
|
||||
da.id,
|
||||
da.dataset_id,
|
||||
da.dataset_files_id,
|
||||
da.question_id,
|
||||
da.answer,
|
||||
da.create_time
|
||||
FROM llm_dataset_answer da
|
||||
WHERE da.deleted = 0
|
||||
AND da.question_id IN
|
||||
<foreach item="item" index="index" collection="collected"
|
||||
open="(" separator="," close=")">
|
||||
#{item}
|
||||
</foreach>
|
||||
</select>
|
||||
</mapper>
|
||||
|
||||
@ -9,4 +9,18 @@
|
||||
文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
|
||||
-->
|
||||
|
||||
</mapper>
|
||||
<select id="getAListOfIssues"
|
||||
resultType="cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO">
|
||||
SELECT
|
||||
id,
|
||||
dataset_id AS datasetId,
|
||||
dataset_files_id AS datasetFilesId,
|
||||
question,
|
||||
status,
|
||||
'system',
|
||||
create_time AS createTime
|
||||
FROM llm_dataset_question
|
||||
WHERE deleted = 0
|
||||
AND dataset_id = #{datasetId}
|
||||
</select>
|
||||
</mapper>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user