DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Csv 解析标注状态修改

Browse Source
This commit is contained in:
Liuyang 2025-01-13 16:40:48 +08:00
parent 0f0ce1a304
commit 9839839037
1 changed files with 46 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
View File

@ -20,6 +20,7 @@ import cn.iocoder.yudao.module.llm.utils.vo.CsvDataSetVO;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
@ -292,6 +293,51 @@ public class DatasetServiceImpl implements DatasetService {
});
}
/**
* csv文件处理
*
* @param csvFiles csv文件
*/
private void readCsvFile (List<DatasetFilesDO> csvFiles) {
csvFiles.forEach(datasetFilesDO -> {
try {
// 读取并解析CSV文件
List<CsvDataSetVO> dataSetVos = DataSetReadFileUtils.readParseCsv(datasetFilesDO.getDatasetFileUrl());
if (CollectionUtils.isNotEmpty(dataSetVos)) {
// 获取数据集ID
Long datasetId = datasetFilesDO.getDatasetId();
// 数据集文件ID
Long fileId = datasetFilesDO.getId();
dataSetVos.forEach(dataSetVO -> {
// 保存到 数据集数据问题
DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
datasetQuestionDO.setDatasetId(datasetId);
datasetQuestionDO.setDatasetFilesId(fileId);
datasetQuestionDO.setSystem(dataSetVO.getSystem());
datasetQuestionDO.setQuestion(dataSetVO.getQuestion());
datasetQuestionDO.setStatus(StringUtils.isNotBlank(dataSetVO.getAnswer()) ? 2 : 0);
datasetQuestionMapper.insert(datasetQuestionDO);
// 保存到 数据集数据问题标注
DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
datasetAnswerDO.setDatasetId(datasetId);
datasetAnswerDO.setDatasetFilesId(fileId);
datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
datasetAnswerDO.setAnswer(dataSetVO.getAnswer());
datasetAnswerMapper.insert(datasetAnswerDO);
});
}
} catch (IOException e) {
throw exception(PARSE_CSV_ERROR);
}
});
}
public void parseFile (List<DatasetFilesSaveReqVO> datasetFiles) {
List<DatasetFilesDO> insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class);
datasetFilesMapper.insertBatch(insertDatasetFiles, 100);
@ -319,47 +365,5 @@ public class DatasetServiceImpl implements DatasetService {
}
}
/**
* csv文件处理
*
* @param csvFiles csv文件
*/
private void readCsvFile (List<DatasetFilesDO> csvFiles) {
csvFiles.forEach(datasetFilesDO -> {
try {
// 读取并解析CSV文件
List<CsvDataSetVO> dataSetVos = DataSetReadFileUtils.readParseCsv(datasetFilesDO.getDatasetFileUrl());
if (CollectionUtils.isNotEmpty(dataSetVos)) {
// 获取数据集ID
Long datasetId = datasetFilesDO.getDatasetId();
// 数据集文件ID
Long fileId = datasetFilesDO.getId();
dataSetVos.forEach(dataSetVO -> {
// 保存到 数据集数据问题
DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
datasetQuestionDO.setDatasetId(datasetId);
datasetQuestionDO.setDatasetFilesId(fileId);
datasetQuestionDO.setSystem(dataSetVO.getSystem());
datasetQuestionDO.setQuestion(dataSetVO.getQuestion());
datasetQuestionMapper.insert(datasetQuestionDO);
// 保存到 数据集数据问题标注
DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
datasetAnswerDO.setDatasetId(datasetId);
datasetAnswerDO.setDatasetFilesId(fileId);
datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
datasetAnswerDO.setAnswer(dataSetVO.getAnswer());
datasetAnswerMapper.insert(datasetAnswerDO);
});
}
} catch (IOException e) {
throw exception(PARSE_CSV_ERROR);
}
});
}
}