package cn.iocoder.yudao.module.llm.service.dataset;


import cn.hutool.core.io.IoUtil;
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.common.exception.ServiceException;
import cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.infra.api.file.FileApi;
import cn.iocoder.yudao.module.llm.constant.DataConstants;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.dto.DataJsonTemplate;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetFilesDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetAnswerMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetFilesMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper;
import cn.iocoder.yudao.module.llm.enums.DatasetStatusEnum;
import cn.iocoder.yudao.module.llm.service.finetuningtask.FineTuningTaskService;
import cn.iocoder.yudao.module.llm.service.modelassesstaskauto.ModelAssessTaskAutoService;
import cn.iocoder.yudao.module.llm.service.modelassesstaskmanual.ModelAssessTaskManualService;
import cn.iocoder.yudao.module.llm.service.modelassesstaskmanualbackup.ModelAssessTaskManualBackupService;
import cn.iocoder.yudao.module.llm.utils.DataSetReadFileUtils;
import cn.iocoder.yudao.module.llm.utils.vo.CsvDataSetVO;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opencsv.exceptions.CsvValidationException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;

import javax.annotation.Resource;
import java.io.*;
import java.net.HttpURLConnection;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.*;

/**
 * 数据集 Service 实现类
 *
 * @author 华大大模型
 */
@Slf4j
@Service
@Validated
public class DatasetServiceImpl implements DatasetService {

    @Resource
    private DatasetMapper datasetMapper;
    @Resource
    private DatasetFilesMapper datasetFilesMapper;
    @Resource
    private DatasetQuestionMapper datasetQuestionMapper;
    @Resource
    private DatasetAnswerMapper datasetAnswerMapper;
    @Resource
    private FileApi fileApi;

    @Resource
    private FineTuningTaskService fineTuningTaskService;

    @Resource
    private ModelAssessTaskManualService modelAssessTaskManualService;

    @Resource
    private ModelAssessTaskAutoService modelAssessTaskAutoService;

    @Resource
    private ModelAssessTaskManualBackupService modelAssessTaskManualBackupService;

    @Transactional
    @Override
    public Long createDataset (DatasetSaveReqVO createReqVO) {
        // 校验
        validateDatasetNameExists(createReqVO);
        if (createReqVO.getType() == null) {
            createReqVO.setType(0);
        }
        // 插入
        DatasetDO dataset = BeanUtils.toBean(createReqVO, DatasetDO.class);
        datasetMapper.insert(dataset);
        List<DatasetFilesSaveReqVO> datasetFiles = createReqVO.getDatasetFiles();
        if (CollectionUtils.isNotEmpty(datasetFiles)) {
            datasetFiles.stream().forEach(
                    datasetFilesSaveReqVO -> {
                        datasetFilesSaveReqVO.setDatasetId(dataset.getId());
                    }
            );
            parseFile(datasetFiles);
            Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
                    .eq(DatasetQuestionDO::getDatasetId, dataset.getId()));

            if (count <= 0){
                throw new ServiceException(new ErrorCode(20000, "数据集问题不能为空"));
            }

            dataset.setDataLength(count);
            Long annoCount = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
                    .eq(DatasetQuestionDO::getDatasetId, dataset.getId())
                    .eq(DatasetQuestionDO::getStatus, 2));
            double ratio = count == 0 ? 0 : ((double) annoCount / count) * 100;
            Integer formattedRatio = ratio == 0 ? 0 : (int) ratio;
            Integer status = formattedRatio == 100 ? 2 : 1;
            if (formattedRatio != null) {
                dataset.setAnnotateProgress(formattedRatio);
            }
            if (annoCount == 0) {
                status = 0;
            }
            if (CollectionUtils.isEmpty(datasetFiles)){
                throw new ServiceException(new ErrorCode(
                        20000, "数据集文件不能为空"));
            }
            if (dataset.getDatasetType() == 2){
                if (status != 2 ){
                    throw new ServiceException(new ErrorCode(
                            20000, "评估数据集只能上传标注完成的数据"));
                }
            }else {
                if (dataset.getStatus() != status) {
                    throw new ServiceException(new ErrorCode(
                            20000, "数据集标注状态错误！应该是【" + DatasetStatusEnum.getStatusByName(status) + "】"));
                }
            }

            dataset.setStatus(status);
            datasetMapper.updateById(dataset);
        }
        return dataset.getId();
    }


    private static long getFileContentLength (File file) throws IOException {
        FileInputStream fis = new FileInputStream(file);
        byte[] buffer = new byte[1024];
        long charCount = 0;
        while (fis.read(buffer) != -1) {
            charCount += new String(buffer).length();
        }
        fis.close();
        return charCount;
    }

    @Override
    @Transactional
    public void updateDataset (DatasetSaveReqVO updateReqVO) {
        // 校验存在
        validateDatasetExists(updateReqVO.getId());
        validateDatasetNameExists(updateReqVO);
        // 更新
        DatasetDO updateObj = BeanUtils.toBean(updateReqVO, DatasetDO.class);
        List<DatasetFilesSaveReqVO> datasetFiles = updateReqVO.getDatasetFiles();
        // 提取id 先过滤掉没有id 再提取
        List<Long> updateIds = datasetFiles.stream().filter(
                datasetFilesSaveReqVO -> datasetFilesSaveReqVO.getId() != null
        ).map(
                datasetFilesSaveReqVO -> datasetFilesSaveReqVO.getId()
        ).collect(Collectors.toList());
        List<DatasetFilesDO> datasetFilesDOS = datasetFilesMapper.selectList(new LambdaQueryWrapper<DatasetFilesDO>()
                .eq(DatasetFilesDO::getDatasetId, updateObj.getId()));
        // 提取之前数据集文件表的id
        List<Long> selectids = datasetFilesDOS.stream().map(
                datasetFilesDO -> datasetFilesDO.getId()
        ).collect(Collectors.toList());

        // 找出需要删除的文件主表id找差集
        List<Long> deleteIds = new ArrayList<>(selectids);
        deleteIds.removeAll(updateIds);
        // 删除操作
        if (CollectionUtils.isNotEmpty(deleteIds)){
            datasetFilesMapper.deleteBatchIds(deleteIds);
            datasetQuestionMapper.delete(new LambdaQueryWrapper<DatasetQuestionDO>()
                    .in(DatasetQuestionDO::getDatasetFilesId, deleteIds));
            datasetAnswerMapper.delete(new LambdaQueryWrapper<DatasetAnswerDO>()
                    .in(DatasetAnswerDO::getDatasetFilesId, deleteIds));
        }
        List<DatasetFilesSaveReqVO> noIdsDatasetFiles = null;

        if (CollectionUtils.isNotEmpty(datasetFiles)) {
            // 提取出没有id的
            noIdsDatasetFiles = datasetFiles.stream().filter(
                    datasetFilesSaveReqVO -> datasetFilesSaveReqVO.getId() == null
            ).collect(Collectors.toList());
            noIdsDatasetFiles.stream().forEach(
                    datasetFilesSaveReqVO -> {
                        datasetFilesSaveReqVO.setDatasetId(updateObj.getId());
                    }
            );
            if (CollectionUtils.isNotEmpty(noIdsDatasetFiles)){
                parseFile(datasetFiles);
            }
            Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
                    .eq(DatasetQuestionDO::getDatasetId, updateObj.getId()));

            if (count <= 0){
                throw new ServiceException(new ErrorCode(20000, "数据集问题不能为空"));
            }

            updateObj.setDataLength(count);
            Long annoCount = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
                    .eq(DatasetQuestionDO::getDatasetId, updateObj.getId())
                    .eq(DatasetQuestionDO::getStatus, 2));
            double ratio = count == 0 ? 0 : ((double) annoCount / count) * 100;
            Integer formattedRatio = ratio == 0 ? 0 : (int) ratio;
            Integer status = formattedRatio == 100 ? 2 : 1;
            if (formattedRatio != null) {
                updateObj.setAnnotateProgress(formattedRatio);
            }
            if (annoCount == 0) {
                status=0;
            }
            if (CollectionUtils.isEmpty(datasetFiles)){
                throw new ServiceException(new ErrorCode(
                        20000, "数据集文件不能为空"));
            }
            if (updateObj.getDatasetType() == 2){
                if (status != 2 ){
                    throw new ServiceException(new ErrorCode(
                            20000, "评估数据集只能上传标注完成的数据"));
                }
            } else {
                if (updateObj.getStatus() != status) {
                    throw new ServiceException(new ErrorCode(
                            20000, "数据集标注状态错误！应该是【" + DatasetStatusEnum.getStatusByName(status) + "】"));
                }
            }
            updateObj.setStatus(status);
        }
        datasetMapper.updateById(updateObj);
    }

    @Override
    public void deleteDataset (Long id) {
        // 校验存在
        validateDatasetExists(id);
        // 校验使用
        validateDatasetUse(id);
        // 删除
        datasetMapper.deleteById(id);
    }

    private void validateDatasetExists (Long id) {
        if (datasetMapper.selectById(id) == null) {
            throw exception(DATASET_NOT_EXISTS);
        }
    }

    /**
     * 校验 数据集 使用情况
     *
     * @param id 数据集ID
     */
    private void validateDatasetUse (Long id) {
        DatasetDO dataset = datasetMapper.selectById(id);
        String datasetName = dataset.getDatasetName();

        // 校验 数据集 是否在 模型调优 有使用 （是否为等待中、进行中）
        validateDatasetUsesInModelTuning(id, datasetName);
        // 校验 数据集 是否在 模型评估任务-人工 有使用 （是否为进行中）
        validateDatasetUsesInModelAssessTaskManual(id, datasetName);
        // 校验 数据集 是否在 模型评估任务-自动 有使用 （是否为进行中、推理中）
        validateDatasetUsesInModelAssessTaskAuto(id, datasetName);
        // 校验 数据集 是否在 模型评估任务备份-人工 有使用
        validateDatasetUsesInModelAssessTaskBackupManual(id, datasetName);
    }

    /**
     * 校验 数据集 是否在 模型调优 有使用
     *
     * @param id          数据集ID
     * @param datasetName 数据集名称
     */
    private void validateDatasetUsesInModelTuning (Long id, String datasetName) {
        Map<Long, String> modelTuning = fineTuningTaskService.getModelTuningByDatasetId(id);
        if (CollectionUtils.isNotEmpty(modelTuning)) {
            String msg = String.format("数据集【%s】在模型调优 %s 有正在等待的任务，请先结束任务", datasetName, modelTuning.values());
            throw exception(new ErrorCode(10002_001, msg));
        }
    }

    /**
     * 校验 数据集 是否在 模型评估任务-人工 有使用
     *
     * @param id          数据集ID
     * @param datasetName 数据集名称
     */
    private void validateDatasetUsesInModelAssessTaskManual (Long id, String datasetName) {
        Map<Long, String> modelAssessmentTask = modelAssessTaskManualService.getModelAssessmentTaskByDatasetId(id);
        if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isNotEmpty(modelAssessmentTask)) {
            String msg = String.format("数据集【%s】在模型评估-人工评估 %s 有正在进行的任务，请先结束任务", datasetName, modelAssessmentTask.values());
            throw exception(new ErrorCode(10002_002, msg));
        }
    }

    /**
     * 校验 数据集 是否在 模型评估任务-自动 有使用
     *
     * @param id        数据集ID
     * @param datasetName 数据集名称
     */
    private void validateDatasetUsesInModelAssessTaskAuto(Long id, String datasetName) {
        Map<Long, String> modelAssessmentTask = modelAssessTaskAutoService.getModelAssessmentTaskByDatasetId(id);
        if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isNotEmpty(modelAssessmentTask)) {
            String msg = String.format("数据集【%s】在模型评估-自动评估 %s 有正在进行的任务，请先结束任务", datasetName, modelAssessmentTask.values());
            throw exception(new ErrorCode(10002_003, msg));
        }
    }

    /**
     * 校验 数据集 是否在 模型评估任务备份-人工 有使用
     *
     * @param id        数据集ID
     * @param datasetName 数据集名称
     */
    private void validateDatasetUsesInModelAssessTaskBackupManual (Long id, String datasetName) {
        Map<Long, String> modelAssessmentTask = modelAssessTaskManualBackupService.getModelAssessmentTaskByDatasetId(id);
        if (CollectionUtils.isNotEmpty(modelAssessmentTask)) {
            String msg = String.format("数据集【%s】在模型评估-人工评估 %s 存在备份，请先取消备份", datasetName, modelAssessmentTask.values());
            throw exception(new ErrorCode(10002_004, msg));
        }
    }

    private void validateDatasetNameExists (DatasetSaveReqVO dateReqVO) {
        LambdaQueryWrapper<DatasetDO> wrapper = new LambdaQueryWrapper<DatasetDO>()
                .eq(DatasetDO::getDatasetName, dateReqVO.getDatasetName());

        if (dateReqVO.getId() != null) {
            wrapper.ne(DatasetDO::getId, dateReqVO.getId());
        }
        List<DatasetDO> datasetDOS = datasetMapper.selectList(wrapper);
        if (CollectionUtils.isNotEmpty(datasetDOS)) {
            throw exception(DATASET_NAME_EXISTS);
        }
    }

    @Override
    public DatasetRespVO getDataset (Long id) {
        DatasetDO datasetDO = datasetMapper.selectById(id);
        DatasetRespVO datasetRespVO = BeanUtils.toBean(datasetDO, DatasetRespVO.class);
        List<DatasetFilesDO> datasetFilesDOS = datasetFilesMapper.selectList(new LambdaQueryWrapper<DatasetFilesDO>().eq(DatasetFilesDO::getDatasetId, id));
        datasetRespVO.setDatasetFiles(BeanUtils.toBean(datasetFilesDOS, DatasetFilesRespVO.class));
        /*List<DatasetQuestionDO> datasetQuestionDO = datasetQuestionMapper.selectList(new LambdaQueryWrapper<DatasetQuestionDO>().eq(DatasetQuestionDO::getDatasetId, id));
        List<DatasetQuestionRespVO> datasetQuestionRespVOS = BeanUtils.toBean(datasetQuestionDO, DatasetQuestionRespVO.class);
        datasetRespVO.setDatasetQuestionRespVOS(datasetQuestionRespVOS);*/
        return datasetRespVO;
    }

    @Override
    public PageResult<DatasetDO> getDatasetPage (DatasetPageReqVO pageReqVO) {
        return datasetMapper.selectPage(pageReqVO);
    }

    @Override
    public List<DatasetTreeNode> queryAll () {
        /*List<DatasetDO> datasetDOS0 = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().eq(DatasetDO::getType, DataConstants.dataTypePrivate));
        List<DatasetRespVO> datasetRespVOS0 = BeanUtils.toBean(datasetDOS0, DatasetRespVO.class);
        List<DatasetDO> datasetDOS1 = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().eq(DatasetDO::getType, DataConstants.dataTypePublic));
        List<DatasetRespVO> datasetRespVOS1 = BeanUtils.toBean(datasetDOS1, DatasetRespVO.class);
        List<List<DatasetRespVO>> result = new ArrayList<>();
        result.add(datasetRespVOS0);
        result.add(datasetRespVOS1);*/
        List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>()
                .eq(DatasetDO::getStatus, 2)); // 获取所有数据集

        // 创建两个根节点，分别代表两种 type
        DatasetTreeNode privateRoot = new DatasetTreeNode(DataConstants.dataTypePrivate);
        DatasetTreeNode publicRoot = new DatasetTreeNode(DataConstants.dataTypePublic);

        for (DatasetDO datasetDO : datasetDOS) {
            DatasetRespVO datasetRespVO = BeanUtils.toBean(datasetDO, DatasetRespVO.class);
            // 根据 type 字段决定节点的位置
            if (datasetRespVO.getType() == DataConstants.dataTypePrivate) {
                privateRoot.getChildren().add(datasetRespVO);
            } else if (datasetRespVO.getType() == DataConstants.dataTypePublic) {
                publicRoot.getChildren().add(datasetRespVO);
            }
        }
        List<DatasetTreeNode> root = new ArrayList<>();
        root.add(privateRoot);
        root.add(publicRoot);
        return root;
    }

    public void readZipFile (List<DatasetFilesDO> zipFiles, List<DatasetFilesSaveReqVO> datasetFiles) {
        DatasetFilesSaveReqVO datasetFilesSaveReqVO = datasetFiles.get(0);
        List<DatasetFilesSaveReqVO> res = new ArrayList<>();
        zipFiles.forEach(datasetFilesDO -> {
            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
            if (connection != null) {
                try {
                    InputStream inputStream = connection.getInputStream();
                    ZipInputStream zipInputStream = new ZipInputStream(inputStream, Charset.forName("GBK"));
                    ZipEntry zipEntry = null;
                    try {
                        while ((zipEntry = zipInputStream.getNextEntry()) != null) {
                            try {
                                final String name = zipEntry.getName();
                                InputStream fileIn = zipInputStream;
                                Map<String, Object> map = fileApi.llmCreateFile(name, IoUtil.readBytes(fileIn, false));
                                String url = map.get("url").toString();
                                Long id = Long.parseLong(map.get("id").toString());
                                DatasetFilesSaveReqVO vo = new DatasetFilesSaveReqVO();
                                vo.setDatasetId(datasetFilesSaveReqVO.getDatasetId());
                                vo.setDatasetFile(id);
                                vo.setDatasetFileUrl(url);
                                vo.setId(datasetFilesSaveReqVO.getId());
                                res.add(vo);
                            } catch (Exception e) {
                                zipInputStream.closeEntry();
                            }
                        }
                    } catch (IOException e) {
                        zipInputStream.close();
                    }
                } catch (Exception e) {
                    throw exception(new ErrorCode(
                            11000, "请正确上传zip格式得数据！！！"));
                } finally {
                    connection.disconnect();
                }
            }
        });
        parseFile(res);
    }

    public void readTarGzFile (List<DatasetFilesDO> tarGzFiles, List<DatasetFilesSaveReqVO> datasetFiles) {
        DatasetFilesSaveReqVO datasetFilesSaveReqVO = datasetFiles.get(0);
        List<DatasetFilesSaveReqVO> res = new ArrayList<>();
        tarGzFiles.forEach(datasetFilesDO -> {
            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
            if (connection != null) {
                try {
                    InputStream inputStream = connection.getInputStream();
                    GzipCompressorInputStream gzipInputStream = new GzipCompressorInputStream(inputStream);
                    TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(gzipInputStream, Charset.forName("GBK").name());
                    TarArchiveEntry tarEntry;
                    try {
                        while ((tarEntry = tarArchiveInputStream.getNextTarEntry()) != null) {
                            if (!tarEntry.isDirectory()) {
                                try {
                                    final String name = tarEntry.getName();
                                    byte[] fileBytes = IoUtil.readBytes(tarArchiveInputStream, false);
                                    Map<String, Object> map = fileApi.llmCreateFile(name, fileBytes);
                                    String url = map.get("url").toString();
                                    Long id = Long.parseLong(map.get("id").toString());
                                    DatasetFilesSaveReqVO vo = new DatasetFilesSaveReqVO();
                                    vo.setDatasetId(datasetFilesSaveReqVO.getDatasetId());
                                    vo.setDatasetFile(id);
                                    vo.setDatasetFileUrl(url);
                                    vo.setId(datasetFilesSaveReqVO.getId());
                                    res.add(vo);
                                } catch (Exception e) {
                                    // Handle exception for individual file
                                }
                            }
                        }
                    } catch (IOException e) {
                        tarArchiveInputStream.close();
                    } finally {
                        tarArchiveInputStream.close();
                    }
                } catch (Exception e) {
                    throw ServiceExceptionUtil.exception(new ErrorCode(
                            11001, "请正确上传tar.gz格式的数据！！！"));
                } finally {
                    connection.disconnect();
                }
            }
        });
        parseFile(res);
    }

    // 暂时先不用
    public void readTarFile (List<DatasetFilesDO> tarFiles, List<DatasetFilesSaveReqVO> datasetFiles) {
        DatasetFilesSaveReqVO datasetFilesSaveReqVO = datasetFiles.get(0);
        List<DatasetFilesSaveReqVO> res = new ArrayList<>();
        tarFiles.forEach(datasetFilesDO -> {
            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
            if (connection != null) {
                try {
                    InputStream inputStream = connection.getInputStream();
                    TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(inputStream);
                    ArchiveEntry archiveEntry = null;
                    try {
                        while ((archiveEntry = tarArchiveInputStream.getNextEntry()) != null) {
                            try {
                                final String name = archiveEntry.getName();
                                InputStream fileIn = tarArchiveInputStream;
                                Map<String, Object> map = fileApi.llmCreateFile(name, IoUtil.readBytes(fileIn));
                                String url = map.get("url").toString();
                                Long id = Long.parseLong(map.get("id").toString());
                                DatasetFilesSaveReqVO vo = new DatasetFilesSaveReqVO();
                                vo.setDatasetId(datasetFilesSaveReqVO.getDatasetId());
                                vo.setDatasetFile(id);
                                vo.setDatasetFileUrl(url);
                                vo.setId(datasetFilesSaveReqVO.getId());
                                res.add(vo);
                            } catch (Exception e) {

                            }
                        }
                    } catch (IOException e) {
                        tarArchiveInputStream.close();
                    }
                } catch (Exception e) {
                    throw exception(new ErrorCode(
                            11000, "请正确上传tar格式得数据！！！"));
                } finally {
                    connection.disconnect();
                }
            }
        });
        parseFile(res);
    }


    public void readJsonFile (List<DatasetFilesDO> jsonFiles) {
        jsonFiles.forEach(datasetFilesDO -> {
            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
            if (connection != null) {
                try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"))) {
                    StringBuilder content = new StringBuilder();
                    String line;
                    while ((line = in.readLine()) != null) {
                        content.append(line);
                    }
                    // 使用Jackson解析 Json 字符串为List<String>对象
                    // 使用Jackson解析 Json 字符串为List<String>对象
                    ObjectMapper mapper = new ObjectMapper();
                    // 使用 TypeReference 解析 JSON 字符串为 List<String>
                    List<DataJsonTemplate> jsonList = mapper.readValue(content.toString(), new TypeReference<List<DataJsonTemplate>>() {
                    });
                    jsonList.forEach(
                            dataJsonTemplate -> {
                                List<String> answers = dataJsonTemplate.getAnswers();
                                DatasetQuestionDO datasetQuestionDO = BeanUtils.toBean(dataJsonTemplate, DatasetQuestionDO.class);
                                datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
                                datasetQuestionDO.setDatasetFilesId(datasetFilesDO.getId());
                                datasetQuestionDO.setStatus(CollectionUtils.isNotEmpty(answers) ? 2 : 0);
                                datasetQuestionMapper.insert(datasetQuestionDO);
                                if (CollectionUtils.isNotEmpty(answers)) {
                                    for (String answer : answers) {
                                        DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
                                        datasetAnswerDO.setDatasetId(datasetFilesDO.getDatasetId());
                                        datasetAnswerDO.setDatasetFilesId(datasetFilesDO.getId());
                                        datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
                                        datasetAnswerDO.setAnswer(answer);
                                        datasetAnswerMapper.insert(datasetAnswerDO);
                                    }
                                }
                            }
                    );
                } catch (Exception e) {
                    throw exception(new ErrorCode(11000, "请正确上传json格式得数据！！！"));
                } finally {
                    connection.disconnect();
                }
            }
        });
    }

    /**
     * txt文本数据
     *
     * @param txtFiles
     */
    public void readTxtFile (List<DatasetFilesDO> txtFiles) {
        txtFiles.forEach(datasetFilesDO -> {
            List<String> newContent = new ArrayList<>();

            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
            if (connection != null) {
                try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"))) {
                    String inputLine;
                    while ((inputLine = in.readLine()) != null) {
                        inputLine = inputLine.trim(); // 去除行首和行尾的空白字符
                        if (!inputLine.isEmpty()) {
                            DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
                            datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
                            datasetQuestionDO.setDatasetFilesId(datasetFilesDO.getId());
                            datasetQuestionDO.setQuestion(inputLine);
                            datasetQuestionMapper.insert(datasetQuestionDO);
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    connection.disconnect();
                }
            }
        });
    }

    /**
     * csv文件处理
     *
     * @param csvFiles csv文件
     */
    private void readCsvFile (List<DatasetFilesDO> csvFiles) {
        csvFiles.forEach(datasetFilesDO -> {

            try {
                // 读取并解析CSV文件
                List<CsvDataSetVO> dataSetVos = DataSetReadFileUtils.readParseCsv(datasetFilesDO.getDatasetFileUrl());

                if (CollectionUtils.isNotEmpty(dataSetVos)) {
                    // 获取数据集ID
                    Long datasetId = datasetFilesDO.getDatasetId();
                    // 数据集文件ID
                    Long fileId = datasetFilesDO.getId();

                    dataSetVos.forEach(dataSetVO -> {
                        boolean answerBlank = StringUtils.isNotBlank(dataSetVO.getAnswer());
                        // 保存到 数据集数据问题
                        DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();// 检查是否为空行
                        datasetQuestionDO.setDatasetId(datasetId);
                        datasetQuestionDO.setDatasetFilesId(fileId);
                        datasetQuestionDO.setSystem(dataSetVO.getSystem());
                        datasetQuestionDO.setQuestion(dataSetVO.getQuestion());
                        datasetQuestionDO.setStatus(answerBlank ? 2 : 0);
                        datasetQuestionMapper.insert(datasetQuestionDO);

                        // 保存到 数据集数据问题标注
                        if (answerBlank) {
                            DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
                            datasetAnswerDO.setDatasetId(datasetId);
                            datasetAnswerDO.setDatasetFilesId(fileId);
                            datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
                            datasetAnswerDO.setAnswer(dataSetVO.getAnswer());
                            datasetAnswerMapper.insert(datasetAnswerDO);
                        }

                    });
                }

            } catch (IOException | CsvValidationException e) {
                throw exception(PARSE_CSV_ERROR);
            }

        });
    }

    /**
     * 解析xlsx文件
     *
     * @param xlsxFiles
     */
    public void readXlsxFile (List<DatasetFilesDO> xlsxFiles) {
        xlsxFiles.forEach(datasetFilesDO -> {
            Workbook sheets = DataSetReadFileUtils.readXlsxFromUrl(datasetFilesDO.getDatasetFileUrl());
            if (sheets != null) {
                Sheet sheetDatas = sheets.getSheetAt(0);
                sheetDatas.forEach(
                        row -> {
                            if (row.getRowNum() != 0) {
                                // 问题
                                DatasetQuestionDO datasetQuestionDO = new DatasetQuestionDO();
                                datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
                                datasetQuestionDO.setDatasetId(datasetFilesDO.getDatasetId());
                                datasetQuestionDO.setDatasetFilesId(datasetFilesDO.getId());
                                // 答案
                                DatasetAnswerDO datasetAnswerDO = new DatasetAnswerDO();
                                datasetAnswerDO.setDatasetId(datasetFilesDO.getDatasetId());
                                datasetAnswerDO.setDatasetFilesId(datasetFilesDO.getId());
                                datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
                                for (Cell cell : row) {
                                    int columnIndex = cell.getColumnIndex();
                                    switch (columnIndex) {
                                        case 0:
                                            datasetQuestionDO.setSystem(cell.getStringCellValue());
                                            break;
                                        case 1:
                                            datasetQuestionDO.setQuestion(cell.getStringCellValue());
                                            break;
                                        case 2:
                                            String cellValue = cell.getStringCellValue();
                                            datasetQuestionDO.setStatus(StringUtils.isNotBlank(cell.getStringCellValue()) ? 2 : 0);
                                            datasetAnswerDO.setAnswer(cellValue);
                                            break;
                                    }
                                }
                                // 插入问题
                                datasetQuestionMapper.insert(datasetQuestionDO);
                                datasetAnswerDO.setQuestionId(datasetQuestionDO.getId());
                                // 插入回答
                                if (StringUtils.isNotBlank(datasetAnswerDO.getAnswer())) {
                                    datasetAnswerMapper.insert(datasetAnswerDO);
                                }
                            }
                        }
                );
            }
        });
    }

    public void parseFile (List<DatasetFilesSaveReqVO> datasetFiles) {
        List<DatasetFilesDO> insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class);
        datasetFilesMapper.insertBatch(insertDatasetFiles, 100);

        // zip文件
        List<DatasetFilesDO> zipFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".zip"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(zipFiles)) {
            readZipFile(zipFiles, datasetFiles);
        }

        // tar文件
        List<DatasetFilesDO> tarFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".tar.gz"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(tarFiles)) {
            readTarGzFile(tarFiles, datasetFiles);
        }

        // 提取文件
        List<DatasetFilesDO> jsonFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".json"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(jsonFiles)) {
            readJsonFile(jsonFiles);
        }

        List<DatasetFilesDO> txtFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".txt"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(txtFiles)) {
            readTxtFile(txtFiles);
        }

        // csv 文件处理
        List<DatasetFilesDO> csvFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".csv"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(csvFiles)) {
            readCsvFile(csvFiles);
        }
        // .xlsx格式
        List<DatasetFilesDO> xlsxFiles = insertDatasetFiles.stream()
                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".xlsx"))
                .collect(Collectors.toList());
        if (CollectionUtils.isNotEmpty(xlsxFiles)) {
            readXlsxFile(xlsxFiles);
        }
    }


}
