From c823592ba0370e345c778ed95829cb464d4dedf2 Mon Sep 17 00:00:00 2001 From: limin Date: Mon, 30 Dec 2024 15:20:26 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=B3=E4=BA=8E=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=20=E6=9B=B4=E6=96=B0=E6=95=B0=E6=8D=AE=E9=9B=86=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=A7=A3=E6=9E=90=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/DatasetQuestionServiceImpl.java | 2 +- .../service/dataset/DatasetServiceImpl.java | 75 ++++++++++++------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java index 4171f1618..4830b3d15 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java @@ -105,7 +105,7 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService { LambdaQueryWrapper wrapper = new LambdaQueryWrapper() .eq(DatasetQuestionDO::getDatasetId, updateReqVOS.get(0).getDatasetId()); Long sumCount = datasetQuestionMapper.selectCount(wrapper); - wrapper.eq(DatasetQuestionDO::getStatus,1); + wrapper.eq(DatasetQuestionDO::getStatus,2); Long annoCount = datasetQuestionMapper.selectCount(wrapper); double ratio = sumCount == 0 ? 0 : ((double) annoCount / sumCount) *100; String formattedRatio = String.format("%.2f", ratio); diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java index fee3fa95a..bf6916c34 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java @@ -59,33 +59,18 @@ public class DatasetServiceImpl implements DatasetService { DatasetDO dataset = BeanUtils.toBean(createReqVO, DatasetDO.class); datasetMapper.insert(dataset); List datasetFiles = createReqVO.getDatasetFiles(); - datasetFiles.stream().forEach( - datasetFilesSaveReqVO -> { - datasetFilesSaveReqVO.setDatasetId(dataset.getId()); - } - ); - List insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class); - System.out.println(insertDatasetFiles); - datasetFilesMapper.insertBatch(insertDatasetFiles, 100); - System.out.println(insertDatasetFiles); - // 提取文件 - List jsonFiles = insertDatasetFiles.stream() - .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".json")) - .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(jsonFiles)){ - readJsonFile(jsonFiles); + if (CollectionUtils.isNotEmpty(datasetFiles)){ + datasetFiles.stream().forEach( + datasetFilesSaveReqVO -> { + datasetFilesSaveReqVO.setDatasetId(dataset.getId()); + } + ); + parseFile(datasetFiles); + Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper() + .eq(DatasetQuestionDO::getDatasetId, dataset.getId())); + dataset.setDataLength(count); + datasetMapper.updateById(dataset); } - - List txtFiles = insertDatasetFiles.stream() - .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".txt")) - .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(txtFiles)){ - readTxtFile(txtFiles); - } - Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper() - .eq(DatasetQuestionDO::getDatasetId, dataset.getId())); - dataset.setDataLength(count); - datasetMapper.updateById(dataset); return dataset.getId(); } @@ -107,6 +92,26 @@ public class DatasetServiceImpl implements DatasetService { validateDatasetExists(updateReqVO.getId()); // 更新 DatasetDO updateObj = BeanUtils.toBean(updateReqVO, DatasetDO.class); + List datasetFiles = updateReqVO.getDatasetFiles(); + if (CollectionUtils.isNotEmpty(datasetFiles)){ + datasetFiles.stream().forEach( + datasetFilesSaveReqVO -> { + datasetFilesSaveReqVO.setDatasetId(updateObj.getId()); + } + ); + parseFile(datasetFiles); + Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper() + .eq(DatasetQuestionDO::getDatasetId, updateObj.getId())); + updateObj.setDataLength(count); + Long annoCount = datasetQuestionMapper.selectCount(new LambdaQueryWrapper() + .eq(DatasetQuestionDO::getDatasetId, updateObj.getId()) + .eq(DatasetQuestionDO::getStatus,2)); + double ratio = count == 0 ? 0 : ((double) annoCount / count) *100; + String formattedRatio = String.format("%.2f%%", ratio); + if (formattedRatio != null){ + updateObj.setAnnotateProgress(formattedRatio); + } + } datasetMapper.updateById(updateObj); } @@ -232,4 +237,22 @@ public class DatasetServiceImpl implements DatasetService { }); } + public void parseFile(List datasetFiles) { + List insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class); + datasetFilesMapper.insertBatch(insertDatasetFiles, 100); + // 提取文件 + List jsonFiles = insertDatasetFiles.stream() + .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".json")) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(jsonFiles)){ + readJsonFile(jsonFiles); + } + + List txtFiles = insertDatasetFiles.stream() + .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".txt")) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(txtFiles)){ + readTxtFile(txtFiles); + } + } }