From 983983903704136db97f30bbf1f07830c6b9a12b Mon Sep 17 00:00:00 2001
From: Liuyang <2746366019@qq.com>
Date: Mon, 13 Jan 2025 16:40:48 +0800
Subject: [PATCH] =?UTF-8?q?Csv=20=E8=A7=A3=E6=9E=90=E6=A0=87=E6=B3=A8?=
 =?UTF-8?q?=E7=8A=B6=E6=80=81=E4=BF=AE=E6=94=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../service/dataset/DatasetServiceImpl.java   | 88 ++++++++++---------
 1 file changed, 46 insertions(+), 42 deletions(-)

diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
index e89b35fd2..dccd3fdfe 100644
--- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
+++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
@@ -20,6 +20,7 @@ import cn.iocoder.yudao.module.llm.utils.vo.CsvDataSetVO;
 import com.alibaba.fastjson.JSON;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.baomidou.mybatisplus.core.toolkit.StringUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
@@ -292,6 +293,51 @@ public class DatasetServiceImpl implements DatasetService {
         });
     }
 
+    /**
+     * csv文件处理
+     *
+     * @param csvFiles csv文件
+     */
+    private void readCsvFile (List<DatasetFilesDO> csvFiles) {
+        csvFiles.forEach(datasetFilesDO -> {
+
+            try {
+                // 读取并解析CSV文件
+                List<CsvDataSetVO> dataSetVos = DataSetReadFileUtils.readParseCsv(datasetFilesDO.getDatasetFileUrl());
+
+                if (CollectionUtils.isNotEmpty(dataSetVos)) {
+                    // 获取数据集ID
+                    Long datasetId = datasetFilesDO.getDatasetId();
+                    // 数据集文件ID
+                    Long fileId = datasetFilesDO.getId();
+
+                    dataSetVos.forEach(dataSetVO -> {
+                        // 保存到 数据集数据问题
+                        DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
+                        datasetQuestionDO.setDatasetId(datasetId);
+                        datasetQuestionDO.setDatasetFilesId(fileId);
+                        datasetQuestionDO.setSystem(dataSetVO.getSystem());
+                        datasetQuestionDO.setQuestion(dataSetVO.getQuestion());
+                        datasetQuestionDO.setStatus(StringUtils.isNotBlank(dataSetVO.getAnswer()) ? 2 : 0);
+                        datasetQuestionMapper.insert(datasetQuestionDO);
+
+                        // 保存到 数据集数据问题标注
+                        DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
+                        datasetAnswerDO.setDatasetId(datasetId);
+                        datasetAnswerDO.setDatasetFilesId(fileId);
+                        datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
+                        datasetAnswerDO.setAnswer(dataSetVO.getAnswer());
+                        datasetAnswerMapper.insert(datasetAnswerDO);
+                    });
+                }
+
+            } catch (IOException e) {
+                throw exception(PARSE_CSV_ERROR);
+            }
+
+        });
+    }
+
     public void parseFile (List<DatasetFilesSaveReqVO> datasetFiles) {
         List<DatasetFilesDO> insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class);
         datasetFilesMapper.insertBatch(insertDatasetFiles, 100);
@@ -319,47 +365,5 @@ public class DatasetServiceImpl implements DatasetService {
         }
     }
 
-    /**
-     * csv文件处理
-     *
-     * @param csvFiles csv文件
-     */
-    private void readCsvFile (List<DatasetFilesDO> csvFiles) {
-        csvFiles.forEach(datasetFilesDO -> {
 
-            try {
-                // 读取并解析CSV文件
-                List<CsvDataSetVO> dataSetVos = DataSetReadFileUtils.readParseCsv(datasetFilesDO.getDatasetFileUrl());
-
-                if (CollectionUtils.isNotEmpty(dataSetVos)) {
-                    // 获取数据集ID
-                    Long datasetId = datasetFilesDO.getDatasetId();
-                    // 数据集文件ID
-                    Long fileId = datasetFilesDO.getId();
-
-                    dataSetVos.forEach(dataSetVO -> {
-                        // 保存到 数据集数据问题
-                        DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
-                        datasetQuestionDO.setDatasetId(datasetId);
-                        datasetQuestionDO.setDatasetFilesId(fileId);
-                        datasetQuestionDO.setSystem(dataSetVO.getSystem());
-                        datasetQuestionDO.setQuestion(dataSetVO.getQuestion());
-                        datasetQuestionMapper.insert(datasetQuestionDO);
-
-                        // 保存到 数据集数据问题标注
-                        DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
-                        datasetAnswerDO.setDatasetId(datasetId);
-                        datasetAnswerDO.setDatasetFilesId(fileId);
-                        datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
-                        datasetAnswerDO.setAnswer(dataSetVO.getAnswer());
-                        datasetAnswerMapper.insert(datasetAnswerDO);
-                    });
-                }
-
-            } catch (IOException e) {
-                throw exception(PARSE_CSV_ERROR);
-            }
-
-        });
-    }
 }