DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

数据表详情增加问题显示详情

Browse Source
This commit is contained in:
limin 2024-12-29 11:20:42 +08:00
parent cb3c903dad
commit 769adc7725
5 changed files with 127 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
yudao-module-llm/yudao-module-llm-api/src/main/java/cn/iocoder/yudao/module/llm/enums/ErrorCodeConstants.java
View File

@ -27,5 +27,8 @@ public interface ErrorCodeConstants {
ErrorCode MODEL_ASSESS_TASK_STOPLIST_NOT_EXISTS = new ErrorCode(10013, "自动评估维度不存在");
ErrorCode MODEL_ASSESS_STOPLIST_NOT_EXISTS = new ErrorCode(10014, "自动评估维度不存在");
ErrorCode LEARNING_RESOURCES_NOT_EXISTS = new ErrorCode(10015, "学习资源不存在");
ErrorCode DATASET_FILES_NOT_EXISTS = new ErrorCode(10016, "数据集文件资源不存在");
ErrorCode DATASET_QUESTION_NOT_EXISTS = new ErrorCode(10017, "数据集标准问题不存在");
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetRespVO.java
View File

@ -56,4 +56,7 @@ public class DatasetRespVO {
@Schema(description = "标注进度", example = "20")
private Integer annotateProgress;
@Schema(description = "数据集数据文件", example = "[]")
private List<DatasetQuestionRespVO> datasetQuestionRespVOS;
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetSaveReqVO.java
View File

@ -44,4 +44,7 @@ public class DatasetSaveReqVO {
@Schema(description = "标注进度", example = "20")
private Integer annotateProgress;
@Schema(description = "数据集数据文件", example = "[]")
private List<DatasetFilesSaveReqVO> datasetFiles;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetService.java
View File

@ -42,7 +42,7 @@ public interface DatasetService {
* @param id 编号
* @return 数据集
*/
DatasetDO getDataset(Long id);
DatasetRespVO getDataset(Long id);
/**
* 获得数据集分页

127
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
View File

@ -1,27 +1,35 @@
package cn.iocoder.yudao.module.llm.service.dataset;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.constant.DataConstants;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetSaveReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetTreeNode;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetFilesDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetFilesMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper;
import cn.iocoder.yudao.module.llm.utils.DataSetReadFileUtils;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.DATASET_NOT_EXISTS;
@ -37,6 +45,10 @@ public class DatasetServiceImpl implements DatasetService {
@Resource
private DatasetMapper datasetMapper;
@Resource
private DatasetFilesMapper datasetFilesMapper;
@Resource
private DatasetQuestionMapper datasetQuestionMapper;
@Override
public Long createDataset(DatasetSaveReqVO createReqVO) {
@ -46,7 +58,34 @@ public class DatasetServiceImpl implements DatasetService {
// 插入
DatasetDO dataset = BeanUtils.toBean(createReqVO, DatasetDO.class);
datasetMapper.insert(dataset);
// 返回
List<DatasetFilesSaveReqVO> datasetFiles = createReqVO.getDatasetFiles();
datasetFiles.stream().forEach(
datasetFilesSaveReqVO -> {
datasetFilesSaveReqVO.setDatasetId(dataset.getId());
}
);
List<DatasetFilesDO> insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class);
System.out.println(insertDatasetFiles);
datasetFilesMapper.insertBatch(insertDatasetFiles, 100);
System.out.println(insertDatasetFiles);
// 提取文件
List<DatasetFilesDO> jsonFiles = insertDatasetFiles.stream()
.filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".json"))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(jsonFiles)){
readJsonFile(jsonFiles);
}
List<DatasetFilesDO> txtFiles = insertDatasetFiles.stream()
.filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".txt"))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(txtFiles)){
readTxtFile(txtFiles);
}
Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
.eq(DatasetQuestionDO::getDatasetId, dataset.getId()));
dataset.setDataLength(count);
datasetMapper.updateById(dataset);
return dataset.getId();
}
@ -86,8 +125,13 @@ public class DatasetServiceImpl implements DatasetService {
}
@Override
public DatasetDO getDataset(Long id) {
return datasetMapper.selectById(id);
public DatasetRespVO getDataset(Long id) {
DatasetDO datasetDO = datasetMapper.selectById(id);
DatasetRespVO datasetRespVO = BeanUtils.toBean(datasetDO, DatasetRespVO.class);
List<DatasetQuestionDO> datasetQuestionDO = datasetQuestionMapper.selectList(new LambdaQueryWrapper<DatasetQuestionDO>().eq(DatasetQuestionDO::getDatasetId, id));
List<DatasetQuestionRespVO> datasetQuestionRespVOS = BeanUtils.toBean(datasetQuestionDO, DatasetQuestionRespVO.class);
datasetRespVO.setDatasetQuestionRespVOS(datasetQuestionRespVOS);
return datasetRespVO;
}
@Override
@ -125,4 +169,67 @@ public class DatasetServiceImpl implements DatasetService {
return root;
}
public void readJsonFile(List<DatasetFilesDO> jsonFiles){
jsonFiles.forEach(datasetFilesDO -> {
HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
if (connection != null){
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"))) {
StringBuilder content = new StringBuilder();
String line;
while ((line = in.readLine()) != null) {
content.append(line);
}
// 使用Jackson解析 Json 字符串为List<String>对象
ObjectMapper mapper = new ObjectMapper();
// 使用 TypeReference 解析 JSON 字符串为 List<String>
List<String> stringList = mapper.readValue(content.toString(), new TypeReference<List<String>>() {});
stringList.forEach(
string -> {
DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();
datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
datasetQuestionDO.setQuestion(string);
datasetQuestionDO.setDatasetFilesId(datasetFilesDO.getId());
datasetQuestionMapper.insert(datasetQuestionDO);
}
);
}catch (Exception e){
throw new RuntimeException("请正确上传json格式得数据！！！");
}finally {
connection.disconnect();
}
}
});
}
/**
* txt文本数据
* @param txtFiles
*/
public void readTxtFile(List<DatasetFilesDO> txtFiles){
txtFiles.forEach(datasetFilesDO -> {
List<String> newContent = new ArrayList<>();
HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
if (connection != null){
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
inputLine = inputLine.trim(); // 去除行首和行尾的空白字符
if (!inputLine.isEmpty()) {
DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
datasetQuestionDO.setDatasetFilesId(datasetFilesDO.getId());
datasetQuestionDO.setQuestion(inputLine);
datasetQuestionMapper.insert(datasetQuestionDO);
}
}
}catch (Exception e){
e.printStackTrace();
}finally {
connection.disconnect();
}
}
});
}
}