feat(llm): 添加数据集和微调模型API配置

- 新增数据集更多模态API地址配置
- 新增微调创建模型路径配置- 完善RAG服务相关注释说明

yudao-module-llm/yudao-module-llm-biz/pom.xml

@@ -19,12 +19,22 @@
     </properties>
 
     <dependencies>
-
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+            <!-- 如果在生产代码中使用，请移除 <scope>test</scope> -->
+            <!-- <scope>test</scope> -->
+        </dependency>
         <dependency>
             <groupId>cn.iocoder.boot</groupId>
             <artifactId>yudao-module-llm-api</artifactId>
             <version>${revision}</version>
         </dependency>
+        <dependency>
+            <groupId>cn.iocoder.boot</groupId>
+            <artifactId>yudao-module-infra-biz</artifactId>
+            <version>2.3.0-jdk8-SNAPSHOT</version>
+        </dependency>
 
         <!-- Web 相关 -->
         <dependency>

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/constant/DataConstants.java

@@ -5,4 +5,5 @@ public class DataConstants {
     // 个人数据集标识
     public static final int dataTypePrivate  = 0;
     public static final int dataTypePublic = 1;
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/constant/DatasetMoreModalConstants.java

@@ -0,0 +1,10 @@
+package cn.iocoder.yudao.module.llm.constant;
+
+public class DatasetMoreModalConstants {
+
+    public static String train_jsonfile="train.json";
+    public static String images_path="images/";
+
+    public static int BUFFER_SIZE = 4096; // 缓冲区大小
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/conversation/ConversationController.java

@@ -110,7 +110,6 @@ public class ConversationController {
     public SseEmitter streamChat (@Valid @RequestBody ChatReqVO chatReqVO, HttpServletResponse response) {
         log.info("收到对话推理请求，请求参数: {}", chatReqVO);
         SseEmitter emitter = new SseEmitter(120_0000L);
-
         // 异步处理，避免阻塞主线程
         CompletableFuture.runAsync(() -> {
             try {
@@ -129,6 +128,29 @@ public class ConversationController {
         return emitter;
     }
 
+    @PostMapping("/stream-chat-modal")
+    public SseEmitter streamChatModal (@Valid @RequestBody ChatReqVO chatReqVO, HttpServletResponse response) {
+        log.info("收到对话推理请求，请求参数: {}", chatReqVO);
+        SseEmitter emitter = new SseEmitter(120_0000L);
+
+        // 异步处理，避免阻塞主线程
+        CompletableFuture.runAsync(() -> {
+            try {
+                conversationService.chatStreamModal(chatReqVO, emitter, response);
+                emitter.complete();
+            } catch (Exception e) {
+                log.error("处理对话推理请求时发生异常:{}", e.getMessage());
+                try {
+                    emitter.completeWithError(e);
+                } catch (Exception ex) {
+                    log.error("无法完成 SseEmitter 错误处理", ex);
+                }
+            }
+        });
+
+        return emitter;
+    }
+
     @GetMapping("/paragraphHitRate")
     @Operation(summary = "获得大模型对话记录分页")
     @PreAuthorize("@ss.hasPermission('llm:conversation:query')")

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/conversation/vo/ChatReqVO.java

@@ -5,6 +5,7 @@ import lombok.Data;
 
 import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
+import java.util.List;
 
 @Schema(description = "管理后台 - 大模型对话推理聊天 Request VO")
 @Data
@@ -35,4 +36,6 @@ public class ChatReqVO {
     private Double topP;
     @Schema(description = "分组id")
     private String groupId;
+
+    private List<String> imagesList;
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/DatasetController.java

@@ -50,6 +50,13 @@ public class DatasetController {
         return success(datasetService.createDataset(createReqVO));
     }
 
+    @PostMapping("/createDatasetMoreModal")
+    @Operation(summary = "创建数据集")
+    //    @PreAuthorize("@ss.hasPermission('llm:dataset:create')")
+    public CommonResult<Long> createDatasetMoreModal (@Valid @RequestBody DatasetSaveReqVO createReqVO) {
+        return success(datasetService.createDatasetMoreModal(createReqVO));
+    }
+
     @PutMapping("/update")
     @Operation(summary = "更新数据集")
     //    @PreAuthorize("@ss.hasPermission('llm:dataset:update')")
@@ -66,6 +73,14 @@ public class DatasetController {
         return success(list);
     }
 
+    @GetMapping("/all/{type}")
+    @Operation(summary = "查询所有数据集接口")
+    //    @PreAuthorize("@ss.hasPermission('llm:dataset:query')")
+    public CommonResult<List<DatasetTreeNode>> queryAllByBaseModelType (@PathVariable("type") Integer type) {
+        List<DatasetTreeNode> list = datasetService.queryAllByBaseModelType(type);
+        return success(list);
+    }
+
     @DeleteMapping("/delete")
     @Operation(summary = "删除数据集")
     @Parameter(name = "id", description = "编号", required = true)
@@ -156,6 +171,8 @@ public class DatasetController {
                 return new FileInfoVO("dataset_example_csv.csv", "text/csv");
             case 4:
                 return new FileInfoVO("dataset_example_json.json", "application/json");
+            case 5:
+                return new FileInfoVO("VL_test.zip", "application/zip");
             default:
                 throw new IllegalArgumentException("无效的 type 参数");
         }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/DatasetQuestionController.java

@@ -1,11 +1,9 @@
 package cn.iocoder.yudao.module.llm.controller.admin.dataset;
 
-import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerRespVO;
-import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionPageReqVO;
-import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
-import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionSaveReqVO;
+import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
 import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
 import cn.iocoder.yudao.module.llm.service.dataset.DatasetQuestionService;
+import cn.iocoder.yudao.module.llm.service.dataset.DatasetService;
 import org.apache.poi.hssf.usermodel.HSSFCellStyle;
 import org.apache.poi.hssf.usermodel.HSSFRow;
 import org.apache.poi.hssf.usermodel.HSSFSheet;
@@ -49,6 +47,9 @@ public class DatasetQuestionController {
     @Resource
     private DatasetQuestionService datasetQuestionService;
 
+    @Resource
+    private DatasetService datasetService;
+
 
     @PutMapping("data-anno")
     @Operation(summary = "保存标注接口")
@@ -71,6 +72,10 @@ public class DatasetQuestionController {
     @ApiAccessLog(operateType = EXPORT)
     public void exportDatasetFilesExcel(@Valid DatasetQuestionPageReqVO pageReqVO,
                                         HttpServletResponse response) throws IOException {
+//        DatasetRespVO dataset = datasetService.getDataset(pageReqVO.getDatasetId());
+//        if(dataset!=null&&dataset.getStatus()!=2){
+//            throw  new RuntimeException("只有状态为已完成的数据才能导出");
+//        }
         HSSFWorkbook template = new HSSFWorkbook();
         HSSFSheet sheet = template.createSheet();
         // 创建样式并设置垂直居中
@@ -94,29 +99,31 @@ public class DatasetQuestionController {
             String system = item.getSystem();
             String question = item.getQuestion();
             List<DatasetAnswerRespVO> datasetAnswerRespVO = item.getDatasetAnswerRespVO();
-            List<String> collect = datasetAnswerRespVO.stream().map(DatasetAnswerRespVO::getAnswer).collect(Collectors.toList());
-            if (collect.size() == 0){
-                row = sheet.createRow(count);
-                row.createCell(0).setCellValue(system);
-                row.getCell(0).setCellStyle(cellStyle);
-                row.createCell(1).setCellValue(question);
-                row.getCell(1).setCellStyle(cellStyle);
-                row.createCell(2).setCellValue("");
-                row.getCell(2).setCellStyle(cellStyle);
-                id.add(count);
-                count++;
-            }else {
-                for (String s : collect) {
+            if(datasetAnswerRespVO!=null&&datasetAnswerRespVO.size()>0){
+                List<String> collect = datasetAnswerRespVO.stream().map(DatasetAnswerRespVO::getAnswer).collect(Collectors.toList());
+                if (collect.size() == 0){
                     row = sheet.createRow(count);
                     row.createCell(0).setCellValue(system);
                     row.getCell(0).setCellStyle(cellStyle);
                     row.createCell(1).setCellValue(question);
                     row.getCell(1).setCellStyle(cellStyle);
-                    row.createCell(2).setCellValue(s);
+                    row.createCell(2).setCellValue("");
                     row.getCell(2).setCellStyle(cellStyle);
+                    id.add(count);
                     count++;
+                }else {
+                    for (String s : collect) {
+                        row = sheet.createRow(count);
+                        row.createCell(0).setCellValue(system);
+                        row.getCell(0).setCellStyle(cellStyle);
+                        row.createCell(1).setCellValue(question);
+                        row.getCell(1).setCellStyle(cellStyle);
+                        row.createCell(2).setCellValue(s);
+                        row.getCell(2).setCellStyle(cellStyle);
+                        count++;
+                    }
+                    id.add(count-1);
                 }
-                id.add(count-1);
             }
         }
         //合并相同内容的单元格

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetPageReqVO.java

@@ -49,4 +49,6 @@ public class DatasetPageReqVO extends PageParam {
     @Schema(description = "标注进度", example = "20")
     private Integer annotateProgress;
 
+    private Integer datasetParentType;
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetQuestionRespVO.java

@@ -40,4 +40,7 @@ public class DatasetQuestionRespVO {
     @Schema(description = "标注内容")
     private List<DatasetAnswerRespVO> datasetAnswerRespVO;
 
+    @Schema(description = "问题对应的图片")
+    private List<String> imagesList;
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetRespVO.java

@@ -68,4 +68,6 @@ public class DatasetRespVO {
     private String jobId;
 
     private String fileUrl;
+
+    private Integer datasetParentType;
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/DatasetSaveReqVO.java

@@ -46,5 +46,7 @@ public class DatasetSaveReqVO {
 
     @Schema(description = "数据集数据文件", example = "[]")
     private List<DatasetFilesSaveReqVO> datasetFiles;
+    @Schema(description = "数据集父类型(1文本数据据,2多模态数据集)", example = "1")
+    private Integer datasetParentType;
 
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/dataset/vo/ZipProcessingResultVo.java

@@ -0,0 +1,23 @@
+package cn.iocoder.yudao.module.llm.controller.admin.dataset.vo;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+@Data
+public class ZipProcessingResultVo implements Serializable {
+
+    private String trailJson;
+
+    private List<Map<String, String>> imagesList;
+
+
+    public ZipProcessingResultVo(String trailJson,List<Map<String, String>> imagesList){
+        this.imagesList=imagesList;
+        this.trailJson=trailJson;
+    }
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/ModelServiceController.java

@@ -108,7 +108,13 @@ public class ModelServiceController {
 //    @PreAuthorize("@ss.hasPermission('llm:model-service:query')")
     public CommonResult<List<ModelServiceRespVO>> getModelServiceList() {
         List<ModelServiceDO> list = modelServiceService.getModelServiceList();
-        return success(BeanUtils.toBean(list, ModelServiceRespVO.class));
+        List<ModelServiceRespVO> modelServiceRespVOS = BeanUtils.toBean(list, ModelServiceRespVO.class);
+        modelServiceRespVOS.stream().forEach(result ->{
+            Long id=result.getFineTuningTask();
+            BaseModelDO basemodel = baseModelService.getById(id);
+            result.setModelType(basemodel.getModelType());
+        });
+        return success(modelServiceRespVOS);
     }
 
     @DeleteMapping("/delete")
@@ -157,6 +163,14 @@ public class ModelServiceController {
         return success(true);
     }
 
+    @PutMapping("/startDatasetMoreModal")
+    @Operation(summary = "启动模型")
+//    @PreAuthorize("@ss.hasPermission('llm:base-model:update')")
+    public CommonResult<Boolean> startDatasetMoreModal(@Valid @RequestBody ModelServiceSaveReqVO updateReqVO) {
+        modelServiceService.startDatasetMoreModal(updateReqVO);
+        return success(true);
+    }
+
     @PutMapping("/disable")
     @Operation(summary = "禁用模型")
 //    @PreAuthorize("@ss.hasPermission('llm:base-model:update')")

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/modelservice/vo/ModelServiceRespVO.java

@@ -78,4 +78,8 @@ public class ModelServiceRespVO {
 
     @Schema(description = "api秘钥")
     private String BaseApiKey;
+
+    @Schema(description = "模型类型")
+    @ExcelProperty("模型类型")
+    private String modelType;
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataset/DatasetAnswerDO.java

@@ -44,4 +44,6 @@ public class DatasetAnswerDO extends BaseDO {
      */
     private String answer;
 
+    private String answerFrom;
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataset/DatasetDO.java

@@ -74,5 +74,7 @@ public class DatasetDO extends BaseDO {
 
     private String fileUrl;
 
+    private Integer datasetParentType;
+
 
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataset/DatasetImagesDO.java

@@ -0,0 +1,48 @@
+package cn.iocoder.yudao.module.llm.dal.dataobject.dataset;
+
+import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
+import com.baomidou.mybatisplus.annotation.KeySequence;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+
+/**
+ * 数据集数据文件 DO
+ *
+ * @author 华大大模型
+ */
+@TableName("llm_dataset_images")
+@KeySequence("llm_dataset_images_seq") // 用于 Oracle、PostgreSQL、Kingbase、DB2、H2 数据库的主键自增。如果是 MySQL 等数据库，可不写。
+@Data
+@EqualsAndHashCode(callSuper = true)
+@ToString(callSuper = true)
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatasetImagesDO extends BaseDO {
+
+    /**
+     * 数据集ID
+     */
+    @TableId
+    private Long id;
+    /**
+     * 数据集ID
+     */
+    private Long datasetId;
+    /**
+     * 数据长度
+     */
+    private Long dataLength;
+    /**
+     * 数据文件（文件表的ID）
+     */
+    private Long datasetFile;
+    /**
+     * 文件URL地址
+     */
+    private String datasetImageUrl;
+
+    private String datasetImageName;
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataset/DatasetQuestionAnswerImageDO.java

@@ -0,0 +1,41 @@
+package cn.iocoder.yudao.module.llm.dal.dataobject.dataset;
+
+import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
+import com.baomidou.mybatisplus.annotation.KeySequence;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+
+/**
+ * 数据集数据文件 DO
+ *
+ * @author 华大大模型
+ */
+@TableName("llm_dataset_question_answer_image")
+@KeySequence("llm_dataset_question_answer_image_seq") // 用于 Oracle、PostgreSQL、Kingbase、DB2、H2 数据库的主键自增。如果是 MySQL 等数据库，可不写。
+@Data
+@EqualsAndHashCode(callSuper = true)
+@ToString(callSuper = true)
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatasetQuestionAnswerImageDO extends BaseDO {
+
+    /**
+     * 数据集ID
+     */
+    @TableId
+    private Long id;
+    /**
+     * 数据集ID
+     */
+    private Long datasetId;
+
+    private Long questionId;
+
+    private Long answerId;
+    private Long dataImageId;
+    private String imageUrl;
+
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/dataset/DatasetQuestionDO.java

@@ -46,4 +46,6 @@ public class DatasetQuestionDO extends BaseDO {
     @TableField("`system`")
     private String system;
 
+    private String questionFrom;
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetImagesMapper.java

@@ -0,0 +1,30 @@
+package cn.iocoder.yudao.module.llm.dal.mysql.dataset;
+
+import cn.iocoder.yudao.framework.common.pojo.PageResult;
+import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
+import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
+import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetFilesPageReqVO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetFilesDO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetImagesDO;
+import org.apache.ibatis.annotations.Mapper;
+
+
+/**
+ * 数据集数据文件 Mapper
+ *
+ * @author 华大大模型
+ */
+@Mapper
+public interface DatasetImagesMapper extends BaseMapperX<DatasetImagesDO> {
+
+//    default PageResult<DatasetFilesDO> selectPage(DatasetFilesPageReqVO reqVO) {
+//        return selectPage(reqVO, new LambdaQueryWrapperX<DatasetFilesDO>()
+//                .eqIfPresent(DatasetFilesDO::getDatasetId, reqVO.getDatasetId())
+//                .eqIfPresent(DatasetFilesDO::getDataLength, reqVO.getDataLength())
+//                .eqIfPresent(DatasetFilesDO::getDatasetFile, reqVO.getDatasetFile())
+//                .eqIfPresent(DatasetFilesDO::getDatasetFileUrl, reqVO.getDatasetFileUrl())
+//                .betweenIfPresent(DatasetFilesDO::getCreateTime, reqVO.getCreateTime())
+//                .orderByDesc(DatasetFilesDO::getId));
+//    }
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetMapper.java

@@ -30,6 +30,7 @@ public interface DatasetMapper extends BaseMapperX<DatasetDO> {
                 .eqIfPresent(DatasetDO::getDatasetFile, reqVO.getDatasetFile())
                 .eqIfPresent(DatasetDO::getDatasetType, reqVO.getDatasetType())
                 .eqIfPresent(DatasetDO::getDatasetFileUrl, reqVO.getDatasetFileUrl())
+                .eq(DatasetDO::getDatasetParentType,reqVO.getDatasetParentType())
                 .betweenIfPresent(DatasetDO::getCreateTime, reqVO.getCreateTime())
                 .orderByDesc(DatasetDO::getId));
     }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/dataset/DatasetQuestionAnswerImageMapper.java

@@ -0,0 +1,27 @@
+package cn.iocoder.yudao.module.llm.dal.mysql.dataset;
+
+import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetImagesDO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionAnswerImageDO;
+import org.apache.ibatis.annotations.Mapper;
+
+
+/**
+ * 数据集数据文件 Mapper
+ *
+ * @author 华大大模型
+ */
+@Mapper
+public interface DatasetQuestionAnswerImageMapper extends BaseMapperX<DatasetQuestionAnswerImageDO> {
+
+//    default PageResult<DatasetFilesDO> selectPage(DatasetFilesPageReqVO reqVO) {
+//        return selectPage(reqVO, new LambdaQueryWrapperX<DatasetFilesDO>()
+//                .eqIfPresent(DatasetFilesDO::getDatasetId, reqVO.getDatasetId())
+//                .eqIfPresent(DatasetFilesDO::getDataLength, reqVO.getDataLength())
+//                .eqIfPresent(DatasetFilesDO::getDatasetFile, reqVO.getDatasetFile())
+//                .eqIfPresent(DatasetFilesDO::getDatasetFileUrl, reqVO.getDatasetFileUrl())
+//                .betweenIfPresent(DatasetFilesDO::getCreateTime, reqVO.getCreateTime())
+//                .orderByDesc(DatasetFilesDO::getId));
+//    }
+
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanual/ManualModelAnswerMapper.java

@@ -30,4 +30,6 @@ public interface ManualModelAnswerMapper extends BaseMapperX<ManualModelAnswerDO
     }
     @Update("update llm_manual_model_answer set status = #{status} where id = #{modelAnswerId}")
     void updateStatus(@Param("modelAnswerId") Long modelAnswerId, @Param("status") int status);
+
+    boolean deleteDataByTaskId(@Param("taskId") Long taskId);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanual/ModelAssessTaskDimensionMapper.java

@@ -9,6 +9,7 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskDimensionPageReqVO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanual.ModelAssessTaskDimensionDO;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 
 /**
@@ -28,4 +29,6 @@ public interface ModelAssessTaskDimensionMapper extends BaseMapperX<ModelAssessT
                 .orderByDesc(ModelAssessTaskDimensionDO::getId));
     }
 
+    public boolean deleteDataByTaskId(@Param("taskId") Long taskId);
+
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanual/ModelAssessTaskManualMapper.java

@@ -31,4 +31,5 @@ public interface ModelAssessTaskManualMapper extends BaseMapperX<ModelAssessTask
 
     @Select("select backup_id from llm_model_assess_task_manual where id = #{id}")
     Long selectTrueById(Long id);
+    boolean deleteByTaskid(@Param("taskId") Long taskId);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanualbackup/ManualModelAnswerBackupMapper.java

@@ -9,6 +9,7 @@ import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanualbackup.
 import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.ManualModelAnswerBackupDO;
 import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 /**
  * 模型评估人工标注 模型回答 Mapper
@@ -31,4 +32,6 @@ public interface ManualModelAnswerBackupMapper extends BaseMapperX<ManualModelAn
     }
     @Delete("delete from llm_manual_model_answer_backup where manal_task_id = #{id}")
     void deleteTrue(Long id);
+
+    boolean deleteByTaskid(@Param("taskId")Long taskId);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanualbackup/ModelAssessTaskDimensionBackupMapper.java

@@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanualbackup.
 import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.ModelAssessTaskDimensionBackupDO;
 import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 /**
  * 人工打分 评估维度 Mapper
@@ -26,4 +27,6 @@ public interface ModelAssessTaskDimensionBackupMapper extends BaseMapperX<ModelA
     }
     @Delete("delete from llm_model_assess_task_dimension_backup where task_id = #{id}")
     void deleteTrue(Long id);
+
+    boolean deleteByTaskid(@Param("taskId")Long taskId);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/modelassesstaskmanualbackup/ModelAssessTaskManualBackupMapper.java

@@ -9,6 +9,7 @@ import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.Mo
 import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Mapper;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanualbackup.vo.*;
+import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Update;
 
 /**
@@ -34,4 +35,6 @@ public interface ModelAssessTaskManualBackupMapper extends BaseMapperX<ModelAsse
     }
     @Delete("delete from llm_model_assess_task_manual_backup where id = #{id}")
     void deleteTrue(Long id);
+
+    boolean deleteByTaskid(@Param("taskId")Long taskId);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java

@@ -75,6 +75,8 @@ public class LLMBackendProperties {
 
     @NotNull(message = "创建微调任务 POST")
     private String finetuningCreate;
+    @NotNull(message = "创建微调任务多模态 POST")
+    private String finetuningCreateModal;
 
     @NotNull(message = "停止微调任务 POST")
     private String stopFinetuning;
@@ -145,4 +147,6 @@ public class LLMBackendProperties {
      * 获取调优检查点列表
      */
     private String checkFileList;
+
+    private String datasetMoreModalApi;
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataSetService.java

@@ -1,14 +1,20 @@
 package cn.iocoder.yudao.module.llm.service.async;
 
 import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
+import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
 import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetFilesDO;
+import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetFilesMapper;
 import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
 import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
+import cn.iocoder.yudao.module.llm.service.dataset.DatasetFilesService;
 import cn.iocoder.yudao.module.llm.service.dataset.vo.AigcDatasetVo;
 import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
 import cn.iocoder.yudao.module.llm.service.http.vo.AigcDatasetFileRespV0;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.StringUtils;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,6 +38,9 @@ public class AsyncDataSetService {
 
     @Resource
     private LLMBackendProperties llmBackendProperties;
+
+    @Resource
+    private DatasetFilesMapper datasetFilesMapper;
     private static final Logger log = LoggerFactory.getLogger(AsyncKnowledgeBase.class);
 
     @Async
@@ -75,6 +84,17 @@ public class AsyncDataSetService {
         }
     }
 
+    @Async
+    public void sendDatasetModal(Long datasetid,String datasetName){
+        List<DatasetFilesDO> datasetFilesDOS = datasetFilesMapper.selectList(Wrappers.<DatasetFilesDO>lambdaQuery().eq(DatasetFilesDO::getDatasetId, datasetid));
+        JSONObject param=new JSONObject();
+        param.put("url",datasetFilesDOS.get(0).getDatasetFileUrl());
+        param.put("dataset_name",datasetName);
+        HttpUtils.post(llmBackendProperties.getDatasetMoreModalApi(),null,param.toString());
+        //llmBackendProperties.getDatasetMoreModalApi()
+
+    }
+
 
     public String JsonFileWriteFine (String hostUrl, DatasetDO datasetDO, List<DatasetQuestionRespVO> datasetQuestionList) {
         try {

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncFineTuningTaskService.java

@@ -122,7 +122,14 @@ public class AsyncFineTuningTaskService {
 
             // 调用模型服务创建微调任务
             log.info("正在调用模型服务创建微调任务...");
-            AigcFineTuningCreateRespVO resp = fineTuningTaskHttpService.finetuningCreate(new HashMap<>(), serverNameDO.getHost(), req);
+            AigcFineTuningCreateRespVO resp=null;
+            String modelType = baseModelDO.getModelType();
+            if("1".equals(modelType)){
+                 resp = fineTuningTaskHttpService.finetuningCreate(new HashMap<>(), serverNameDO.getHost(), req);
+            }else{
+                resp = fineTuningTaskHttpService.finetuningCreateModal(new HashMap<>(), serverNameDO.getHost(), req);
+            }
+
 
             // 更新任务状态
             FineTuningTaskDO updateObj = new FineTuningTaskDO();

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncModelAccessManualService.java

@@ -45,6 +45,7 @@ import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -292,6 +293,7 @@ public class AsyncModelAccessManualService {
                     ModelAssessTaskDimensionBackupDO bean = BeanUtils.toBean(item, ModelAssessTaskDimensionBackupDO.class);
                     bean.setId(null);
                     bean.setTaskId(backupDo.getId());
+                    bean.setCreateTime(LocalDateTime.now());
                     modelAssessTaskDimensionBackupMapper.insert(bean);
                 });
             }
@@ -302,6 +304,7 @@ public class AsyncModelAccessManualService {
                     ManualModelAnswerBackupDO bean = BeanUtils.toBean(item, ManualModelAnswerBackupDO.class);
                     bean.setId(null);
                     bean.setManalTaskId(backupDo.getId());
+                    bean.setCreateTime(LocalDateTime.now());
                     manualModelAnswerBackupMapper.insert(bean);
                 });
             }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelServiceImpl.java

@@ -292,7 +292,7 @@ public class BaseModelServiceImpl implements BaseModelService {
     @Override
     public List<BaseModelDO> listBaseModels() {
         LambdaQueryWrapper<BaseModelDO> select = new LambdaQueryWrapper<BaseModelDO>()
-                .select(BaseModelDO::getId, BaseModelDO::getModelName, BaseModelDO::getTheTuningName);
+                .select(BaseModelDO::getId, BaseModelDO::getModelName, BaseModelDO::getTheTuningName,BaseModelDO::getModelType);
 
         List<BaseModelDO> selects = baseModelMapper.selectList(select);
 

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java

@@ -24,6 +24,7 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.esotericsoftware.minlog.Log;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -260,6 +261,12 @@ public class BaseModelTaskService {
                     BaseModelDO baseModelDO = new BaseModelDO();
                     baseModelDO.setModelName(remoteModelName);
                     baseModelDO.setAigcModelName(remoteModelName);
+                    String[] split = remoteModelName.split("-" + active);
+                    String oldbasemodelname=split[0];
+                    BaseModelDO oldbasemodel = baseModelMapper.selectOne(Wrappers.<BaseModelDO>lambdaQuery().eq(BaseModelDO::getModelName, oldbasemodelname));
+                    if(oldbasemodel!=null){
+                        baseModelDO.setModelType(oldbasemodel.getModelType());
+                    }
                     // 模型类型
                     // 微调状态
                     boolean b = replaceActiveGroups(remoteModelName, active);

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationService.java

@@ -75,6 +75,8 @@ public interface ConversationService {
      */
     void chatStream (@Valid ChatReqVO chatReqVO, SseEmitter emitter, HttpServletResponse response);
 
+    void chatStreamModal(ChatReqVO chatReqVO, SseEmitter emitter, HttpServletResponse response);
+
     /**
      * 获取段落命中率
      * @param uuid

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/conversation/ConversationServiceImpl.java

@@ -13,11 +13,9 @@ import cn.iocoder.yudao.module.llm.controller.admin.application.vo.ApplicationSa
 import cn.iocoder.yudao.module.llm.controller.admin.conversation.vo.*;
 import cn.iocoder.yudao.module.llm.controller.admin.conversation.vo.ChatReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.datarefluxdata.vo.DataRefluxDataSaveReqVO;
-import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBaseSaveReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestResultVO;
 import cn.iocoder.yudao.module.llm.controller.admin.prompttemplates.vo.PromptTemplatesRespVO;
-import cn.iocoder.yudao.module.llm.controller.admin.prompttemplates.vo.PromptTemplatesSaveReqVO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.conversation.ConversationDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
@@ -341,6 +339,49 @@ public class ConversationServiceImpl implements ConversationService {
         publicModelChatStream(chatReqVO, emitter);
     }
 
+    @Override
+    public void chatStreamModal(ChatReqVO chatReqVO, SseEmitter emitter, HttpServletResponse response) {
+        log.info("开始处理对话请求，请求参数: {}", chatReqVO);
+        // 检查系统提示信息，如果为空则尝试从应用中获取
+        if (chatReqVO.getSystemPrompt() == null || chatReqVO.getSystemPrompt().isEmpty()) {
+            if (chatReqVO.getApplicationId() != null) {
+                log.info("系统提示信息为空，尝试从应用中获取，应用 ID: {}", chatReqVO.getApplicationId());
+                ApplicationRespVO application = applicationService.getApplication(chatReqVO.getApplicationId());
+                List<String> messageHistoryList = stringRedisTemplate.opsForList().range(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), 0, -1);
+                if (CollectionUtils.isEmpty(messageHistoryList)) {
+                    log.info("聊天历史记录为空，更新应用聊天计数");
+                    application.setChatCount(application.getChatCount() + 1);
+                    ApplicationSaveReqVO applicationSaveReqVO = BeanUtil.toBean(application, ApplicationSaveReqVO.class);
+                    applicationService.updateApplication(applicationSaveReqVO);
+                }
+
+
+                // 知识库ID
+                if (application.getModelServiceId() != null) {
+                    chatReqVO.setKnowledge(application.getModelServiceId());
+                }
+
+                if (!org.apache.commons.lang3.StringUtils.isBlank(application.getPrompt())) {
+                    chatReqVO.setSystemPrompt(application.getPrompt());
+                    if (chatReqVO.getSystemPrompt() == null || chatReqVO.getSystemPrompt().isEmpty()) {
+                        log.info("应用中未找到系统提示信息，使用默认提示信息");
+                        chatReqVO.setSystemPrompt(PROMPT);
+                    }
+                }
+
+                Optional.ofNullable(application.getPromptId()).ifPresent(promptId -> {
+                    PromptTemplatesRespVO promptTemplates = promptTemplatesService.getPromptTemplates(promptId);
+                    int count = Math.max(1, Optional.ofNullable(promptTemplates.getUseCount()).orElse(0) + 1);
+                    promptTemplatesService.updatePromptTemplatesUseCount(Math.toIntExact(promptId),count);
+                });
+
+                log.info("已更新系统提示信息为: {}", chatReqVO.getSystemPrompt());
+            }
+        }
+        // 调用公共模型聊天流式处理方法
+        publicModelChatStreamModal(chatReqVO, emitter);
+    }
+
     /**
      * 获取段落命中率
      *
@@ -547,6 +588,187 @@ public class ConversationServiceImpl implements ConversationService {
         log.info("数据回流信息保存完成");
     }
 
+    public void publicModelChatStreamModal(ChatReqVO chatReqVO, SseEmitter emitter) {
+        log.info("开始公共模型聊天流式处理，请求参数: {}", chatReqVO);
+        // 检查 UUID 是否为空，若为空则生成一个
+        String uuid = chatReqVO.getUuid();
+        if (StrUtil.isBlank(uuid)) {
+            log.info("UUID 为空，生成新的 UUID");
+            uuid = UUID.randomUUID().toString();
+            chatReqVO.setUuid(uuid);
+        }
+
+        // 为每一次对话设置ID
+        String groupId = UUID.randomUUID().toString();
+        chatReqVO.setGroupId(groupId);
+
+        String model = null;
+        String selfModelUrl = "";
+        // 根据模型类型获取模型信息
+        if (Objects.equals(1, chatReqVO.getModelType())) {
+            log.info("使用预制模型，模型 ID: {}", chatReqVO.getModelId());
+            BaseModelDO baseModelDO = baseModelService.getBaseModel(chatReqVO.getModelId());
+            if (baseModelDO == null) {
+                log.error("预制模型不存在，模型 ID: {}", chatReqVO.getModelId());
+                try {
+                    emitter.completeWithError(new RuntimeException("BASE_MODEL_NOT_EXISTS"));
+                } catch (Exception e) {
+                    log.error("无法完成 SseEmitter 错误处理", e);
+                }
+                return;
+            }
+            selfModelUrl = baseModelDO.getChatUrl();
+            model = baseModelDO.getAigcModelName();
+            log.info("获取到预制模型信息，模型名称: {}, 聊天 URL: {}", model, selfModelUrl);
+        } else if (Objects.equals(0, chatReqVO.getModelType())) {
+            log.info("使用自定义模型，模型 ID: {}", chatReqVO.getModelId());
+            ModelServiceDO modelServiceDO = modelServiceMapper.selectById(chatReqVO.getModelId());
+            if (modelServiceDO == null) {
+                log.error("自定义模型服务不存在，模型 ID: {}", chatReqVO.getModelId());
+                try {
+                    emitter.completeWithError(new RuntimeException("MODEL_SERVICE_NOT_EXISTS"));
+                } catch (Exception e) {
+                    log.error("无法完成 SseEmitter 错误处理", e);
+                }
+                return;
+            }
+            model = modelServiceDO.getBaseModelName();
+            selfModelUrl = modelServiceDO.getModelUrl();
+            log.info("获取到自定义模型信息，模型名称: {}, 模型 URL: {}", model, selfModelUrl);
+        } else {
+            log.error("无效的模型类型，模型类型: {}", chatReqVO.getModelType());
+            try {
+                emitter.completeWithError(new RuntimeException("BASE_MODEL_NOT_EXISTS"));
+            } catch (Exception e) {
+                log.error("无法完成 SseEmitter 错误处理", e);
+            }
+            return;
+        }
+
+        List<ModelCompletionsModalReqVO.ModelCompletionsMessageModal> messages = new ArrayList<>();
+        String systemPrompt = "";
+        String knowledgeBaseString = "";
+        if (chatReqVO.getKnowledge() != null) {
+            StringBuilder knowledgeBase = getKnowledgeBase(chatReqVO);
+            knowledgeBaseString = knowledgeBase.toString();
+
+            // 处理 knowledgeBaseString
+            if (StringUtils.isNotBlank(knowledgeBaseString)) {
+                knowledgeBaseString = "<context>" + knowledgeBaseString + "</context>";
+            } else {
+                knowledgeBaseString = "<context>" + "</context>";
+            }
+
+            // 处理 systemPrompt
+            systemPrompt = StringUtils.isBlank(chatReqVO.getSystemPrompt())
+                    ? PROMPT
+                    : chatReqVO.getSystemPrompt() + "   \n   " + PROMPT;
+        }
+        String mess = systemPrompt + "   \n   " + knowledgeBaseString;
+
+        if (chatReqVO.getKnowledge() != null) {
+            log.info("不存在聊天历史记录，创建新的系统消息");
+            ModelCompletionsModalReqVO.ModelCompletionsMessageModal systemMessage = new ModelCompletionsModalReqVO.ModelCompletionsMessageModal();
+            systemMessage.setRole("system");
+            com.alibaba.fastjson.JSONArray jsonarray=new com.alibaba.fastjson.JSONArray();
+            jsonarray.add(mess);
+            systemMessage.setContent(jsonarray);
+            stringRedisTemplate.opsForList().rightPush(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), JsonUtils.toJsonString(systemMessage));
+//            messages.add(systemMessage);
+        } else {
+            // 查询历史记录消息，并将查询出来的知识信息放入到 role = system 的消息中
+            List<String> messageHistoryList = stringRedisTemplate.opsForList().range(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), 0, -1);
+            if (messageHistoryList != null && !messageHistoryList.isEmpty()) {
+                log.info("存在聊天历史记录，处理历史记录消息");
+                for (String messageHistory : messageHistoryList) {
+                    ModelCompletionsModalReqVO.ModelCompletionsMessageModal modelCompletionsMessage = JsonUtils.parseObject(messageHistory, ModelCompletionsModalReqVO.ModelCompletionsMessageModal.class);
+                    if ("system".equals(modelCompletionsMessage.getRole())) {
+                        com.alibaba.fastjson.JSONArray jsonarray=new com.alibaba.fastjson.JSONArray();
+                        JSONObject contentjson=new JSONObject(true);
+                        contentjson.put("text",mess);
+                        jsonarray.add(contentjson);
+                        modelCompletionsMessage.setContent(jsonarray);
+                        stringRedisTemplate.opsForList().set(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), 0, JsonUtils.toJsonString(modelCompletionsMessage));
+                    }
+                    messages.add(modelCompletionsMessage);
+                }
+            }
+//            else {
+//                log.info("不存在聊天历史记录，创建新的系统消息");
+//                ModelCompletionsReqVO.ModelCompletionsMessage systemMessage = new ModelCompletionsReqVO.ModelCompletionsMessage();
+//                systemMessage.setRole("system");
+//                systemMessage.setContent(mess);
+//                stringRedisTemplate.opsForList().rightPush(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), JsonUtils.toJsonString(systemMessage));
+//                messages.add(systemMessage);
+//            }
+        }
+
+
+        // 创建用户消息
+        ModelCompletionsModalReqVO.ModelCompletionsMessageModal message = new ModelCompletionsModalReqVO.ModelCompletionsMessageModal();
+        message.setRole("user");
+        com.alibaba.fastjson.JSONArray contentJsonArray=new com.alibaba.fastjson.JSONArray();
+        JSONObject textcontentJson=new JSONObject();
+        textcontentJson.put("type","text");
+        String msg="";
+        if(!"".equals(systemPrompt)){
+            msg=msg+systemPrompt;
+        }
+        if(!"".equals(knowledgeBaseString)){
+            msg=msg+"知识信息:{"+knowledgeBaseString+"}";
+        }
+        textcontentJson.put("text",msg+chatReqVO.getPrompt());
+        contentJsonArray.add(textcontentJson);
+        List<String> imagesList = chatReqVO.getImagesList();
+        if(imagesList!=null&&imagesList.size()>0){
+            JSONObject imagecontentJson=new JSONObject();
+            imagecontentJson.put("type","image_url");
+            JSONObject imageurljson=new JSONObject();
+            imageurljson.put("url",imagesList.get(0));
+            imagecontentJson.put("image_url",imageurljson);
+            contentJsonArray.add(imagecontentJson);
+        }
+        message.setContent(contentJsonArray);
+        messages.add(message);
+
+        // 构建模型补全请求对象
+        ModelCompletionsModalReqVO modelCompletionsReqVO = new ModelCompletionsModalReqVO();
+        modelCompletionsReqVO.setMessages(messages);
+        modelCompletionsReqVO.setModel(model);
+        modelCompletionsReqVO.setTemperature(chatReqVO.getTemperature());
+        modelCompletionsReqVO.setTop_p(chatReqVO.getTopP());
+        modelCompletionsReqVO.setMax_tokens(chatReqVO.getMaxTokens());
+        //        log.info("构建模型补全请求对象，请求参数1: {}", modelCompletionsReqVO);
+        System.out.println("-----------------整体请求参数"+JSONObject.toJSON(modelCompletionsReqVO));
+        // 调用模型服务进行流式处理
+        ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletionsStreamModal(selfModelUrl, modelCompletionsReqVO, emitter, chatReqVO.getUuid(), chatReqVO.getGroupId(), knowledgeBaseString);
+        if (modelCompletionsRespVO == null) {
+            throw exception(MODEL_COMPLETIONS_ERROR);
+        }
+        // 将用户消息存入缓存
+        stringRedisTemplate.opsForList().rightPush(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), JsonUtils.toJsonString(message));
+        ModelCompletionsModalReqVO.ModelCompletionsMessageModal responseMessage = new ModelCompletionsModalReqVO.ModelCompletionsMessageModal();
+        responseMessage.setRole("assistant");
+        com.alibaba.fastjson.JSONArray contentjsonarray=new com.alibaba.fastjson.JSONArray();
+        JSONObject answerjson=new JSONObject(true);
+        answerjson.put("text",modelCompletionsRespVO.getAnswer());
+        answerjson.put("type","text");
+        contentjsonarray.add(answerjson);
+        responseMessage.setContent(contentjsonarray);
+        stringRedisTemplate.opsForList().rightPush(CHAT_HIStORY_REDIS_KEY + ":" + chatReqVO.getUuid(), JsonUtils.toJsonString(responseMessage));
+        // 保存数据回流信息
+        DataRefluxDataSaveReqVO dataRefluxDataSaveReqVO = new DataRefluxDataSaveReqVO();
+        dataRefluxDataSaveReqVO.setModelServiceId(chatReqVO.getModelId());
+        dataRefluxDataSaveReqVO.setModelType(chatReqVO.getModelType());
+        dataRefluxDataSaveReqVO.setPrompt(chatReqVO.getPrompt());
+        dataRefluxDataSaveReqVO.setSystem(modelCompletionsRespVO.getSystem());
+        dataRefluxDataSaveReqVO.setResponse(modelCompletionsRespVO.getAnswer().replaceAll("(?s)<think>.*?</think>", "").trim());
+        dataRefluxDataSaveReqVO.setMaxTokens(chatReqVO.getMaxTokens());
+        dataRefluxDataSaveReqVO.setTemperature(chatReqVO.getTemperature());
+        dataRefluxDataService.saveDataRefluxData(dataRefluxDataSaveReqVO);
+        log.info("数据回流信息保存完成");
+    }
+
     @NotNull
     private StringBuilder getKnowledgeBase(ChatReqVO chatReqVO) {
         final String LOG_PREFIX = "[KnowledgeBase]";
@@ -582,7 +804,9 @@ public class ConversationServiceImpl implements ConversationService {
 
             log.info("{} 知识库构建完成，内容长度: {}", LOG_PREFIX, knowledgeBase.length());
         } catch (Exception e) {
+
             log.error("{} 知识库处理异常: {}", LOG_PREFIX, e.getMessage(), e);
+            e.printStackTrace();
             throw new ServiceException(100, "知识库处理失败，请稍后重试");
         }
 

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetQuestionServiceImpl.java

@@ -2,12 +2,16 @@ package cn.iocoder.yudao.module.llm.service.dataset;
 
 import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
 import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionAnswerImageDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
 import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetAnswerMapper;
 import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
+import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionAnswerImageMapper;
 import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import jodd.util.StringUtil;
 import org.springframework.stereotype.Service;
 
@@ -43,6 +47,9 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
     @Resource
     private DatasetMapper datasetMapper;
 
+    @Resource
+    private DatasetQuestionAnswerImageMapper datasetQuestionAnswerImageMapper;
+
     @Override
     public Long createDatasetQuestion(DatasetQuestionSaveReqVO createReqVO) {
         // 插入
@@ -83,14 +90,39 @@ public class DatasetQuestionServiceImpl implements DatasetQuestionService {
     @Override
     public PageResult<DatasetQuestionRespVO> getDatasetQuestionPage(DatasetQuestionPageReqVO pageReqVO) {
         PageResult<DatasetQuestionDO> datasetQuestionDOPageResult = datasetQuestionMapper.selectPage(pageReqVO);
+        Long datasetId = pageReqVO.getDatasetId();
+        DatasetDO datasetDO = datasetMapper.selectById(datasetId);
+        Integer datasetParentType = datasetDO.getDatasetParentType();
         PageResult<DatasetQuestionRespVO> result = BeanUtils.toBean(datasetQuestionDOPageResult, DatasetQuestionRespVO.class);
         if (CollectionUtils.isNotEmpty(result.getList())) {
-            result.getList().forEach(item -> {
-                List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
-                        .eq(DatasetAnswerDO::getQuestionId, item.getId()));
-                item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class));
+//            result.getList().forEach(item -> {
+//                List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class)
+//                        .eq(DatasetAnswerDO::getQuestionId, item.getId()));
+//                item.setDatasetAnswerRespVO(BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class));
+//
+//            });
+            // 优化代码
+            List<DatasetQuestionRespVO> list = result.getList();
+            // 获取答案
+            Set<Long> collect = list.stream().map(DatasetQuestionRespVO::getId).collect(Collectors.toSet());
+            LambdaQueryWrapper<DatasetAnswerDO> wrapper = new LambdaQueryWrapper<DatasetAnswerDO>()
+                    .in(DatasetAnswerDO::getQuestionId, collect);
+            List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(wrapper);
+            List<DatasetAnswerRespVO> respVOS = BeanUtils.toBean(datasetAnswerDOS, DatasetAnswerRespVO.class);
+            Map<Long, List<DatasetAnswerRespVO>> collect1 = respVOS.stream().collect(Collectors.groupingBy(DatasetAnswerRespVO::getQuestionId));
+            list.forEach(item -> {
+                item.setDatasetAnswerRespVO(collect1.get(item.getId()));
+                if(datasetParentType==2){
+                    LambdaQueryWrapper<DatasetQuestionAnswerImageDO> imagewrapper = new LambdaQueryWrapper<DatasetQuestionAnswerImageDO>()
+                            .eq(DatasetQuestionAnswerImageDO::getQuestionId, item.getId())
+                            .eq(DatasetQuestionAnswerImageDO::getDatasetId,item.getDatasetId());
 
+                    List<DatasetQuestionAnswerImageDO> datasetQuestionAnswerImageDOList = datasetQuestionAnswerImageMapper.selectList(imagewrapper);
+                    List<String> imageUrlList = datasetQuestionAnswerImageDOList.stream().map(DatasetQuestionAnswerImageDO::getImageUrl).collect(Collectors.toList());
+                    item.setImagesList(imageUrlList);
+                }
             });
+
         }
         return result;
     }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetService.java

@@ -8,6 +8,7 @@ import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
 import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
 import cn.iocoder.yudao.framework.common.pojo.PageResult;
 import cn.iocoder.yudao.framework.common.pojo.PageParam;
+import org.springframework.transaction.annotation.Transactional;
 
 /**
  * 数据集 Service 接口
@@ -24,6 +25,9 @@ public interface DatasetService {
      */
     Long createDataset(@Valid DatasetSaveReqVO createReqVO);
 
+    @Transactional
+    Long createDatasetMoreModal(DatasetSaveReqVO createReqVO);
+
     /**
      * 更新数据集
      *
@@ -57,4 +61,5 @@ public interface DatasetService {
     List<DatasetTreeNode>  queryAll();
 
 
+    List<DatasetTreeNode> queryAllByBaseModelType(Integer type);
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java

@@ -8,17 +8,13 @@ import cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil;
 import cn.iocoder.yudao.framework.common.pojo.PageResult;
 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
 import cn.iocoder.yudao.module.infra.api.file.FileApi;
+import cn.iocoder.yudao.module.infra.service.file.FileService;
 import cn.iocoder.yudao.module.llm.constant.DataConstants;
+import cn.iocoder.yudao.module.llm.constant.DatasetMoreModalConstants;
 import cn.iocoder.yudao.module.llm.controller.admin.dataset.dto.DataJsonTemplate;
 import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.*;
-import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetAnswerDO;
-import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
-import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetFilesDO;
-import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetQuestionDO;
-import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetAnswerMapper;
-import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetFilesMapper;
-import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
-import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionMapper;
+import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.*;
+import cn.iocoder.yudao.module.llm.dal.mysql.dataset.*;
 import cn.iocoder.yudao.module.llm.enums.DatasetStatusEnum;
 import cn.iocoder.yudao.module.llm.service.finetuningtask.FineTuningTaskService;
 import cn.iocoder.yudao.module.llm.service.modelassesstaskauto.ModelAssessTaskAutoService;
@@ -26,6 +22,8 @@ import cn.iocoder.yudao.module.llm.service.modelassesstaskmanual.ModelAssessTask
 import cn.iocoder.yudao.module.llm.service.modelassesstaskmanualbackup.ModelAssessTaskManualBackupService;
 import cn.iocoder.yudao.module.llm.utils.DataSetReadFileUtils;
 import cn.iocoder.yudao.module.llm.utils.vo.CsvDataSetVO;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
 import com.baomidou.mybatisplus.core.toolkit.StringUtils;
@@ -41,6 +39,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
+import org.springframework.mock.web.MockMultipartFile;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.validation.annotation.Validated;
@@ -49,9 +48,12 @@ import javax.annotation.Resource;
 import java.io.*;
 import java.net.HttpURLConnection;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -92,6 +94,20 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private ModelAssessTaskManualBackupService modelAssessTaskManualBackupService;
 
+
+    @Resource
+    private DatasetImagesMapper datasetImagesMapper;
+    @Resource
+    private FileService fileService;
+
+    @Resource
+    private DatasetQuestionAnswerImageMapper datasetQuestionAnswerImageMapper;
+
+    private static final Pattern IMAGE_FILE_EXTENSION_PATTERN = Pattern.compile("\\.([^.]+)$");
+
+    //json文件中的地址前缀都有一个image/;
+    private static final String JSON_IMAGE_PERFEX="image/";
+
     private static long getFileContentLength(File file) throws IOException {
         FileInputStream fis = new FileInputStream(file);
         byte[] buffer = new byte[1024];
@@ -164,6 +180,320 @@ public class DatasetServiceImpl implements DatasetService {
         return dataset.getId();
     }
 
+
+
+
+
+
+    /**多模让i数据集创建
+     *
+     * */
+    @Override
+    @Transactional
+    public Long createDatasetMoreModal(DatasetSaveReqVO createReqVO) {
+        // 校验
+        validateDatasetNameExists(createReqVO);
+        if (createReqVO.getType() == null) {
+            createReqVO.setType(0);
+        }
+        List<DatasetFilesSaveReqVO> datasetFiles = createReqVO.getDatasetFiles();
+        // 插入
+        DatasetDO dataset = BeanUtils.toBean(createReqVO, DatasetDO.class);
+        dataset.setStatus(2);
+        datasetMapper.insert(dataset);
+
+        if (CollectionUtils.isNotEmpty(datasetFiles)) {
+            datasetFiles.stream().forEach(
+                    datasetFilesSaveReqVO -> {
+                        datasetFilesSaveReqVO.setDatasetId(dataset.getId());
+                    }
+            );
+//            parseFile(datasetFiles);
+            resolveZipFileDatasetMoreModal(datasetFiles);
+
+            Long count = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
+                    .eq(DatasetQuestionDO::getDatasetId, dataset.getId()));
+
+            if (count <= 0) {
+                throw new ServiceException(new ErrorCode(20000, "数据集问题不能为空"));
+            }
+
+            dataset.setDataLength(count);
+            Long annoCount = datasetQuestionMapper.selectCount(new LambdaQueryWrapper<DatasetQuestionDO>()
+                    .eq(DatasetQuestionDO::getDatasetId, dataset.getId())
+                    .eq(DatasetQuestionDO::getStatus, 2));
+            double ratio = count == 0 ? 0 : ((double) annoCount / count) * 100;
+            Integer formattedRatio = ratio == 0 ? 0 : (int) ratio;
+            Integer status = formattedRatio == 100 ? 2 : 1;
+            if (formattedRatio != null) {
+                dataset.setAnnotateProgress(formattedRatio);
+            }
+            if (annoCount == 0) {
+                status = 0;
+            }
+            if (CollectionUtils.isEmpty(datasetFiles)) {
+                throw new ServiceException(new ErrorCode(
+                        20000, "数据集文件不能为空"));
+            }
+//            if (dataset.getDatasetType() == 2) {
+//                if (status != 2) {
+//                    throw new ServiceException(new ErrorCode(
+//                            20000, "评估数据集只能上传标注完成的数据"));
+//                }
+//            } else {
+//                if (dataset.getStatus() != status) {
+//                    throw new ServiceException(new ErrorCode(
+//                            20000, "数据集标注状态错误！应该是【" + DatasetStatusEnum.getStatusByName(status) + "】"));
+//                }
+//            }
+
+            dataset.setStatus(status);
+            datasetMapper.updateById(dataset);
+        }
+        return dataset.getId();
+    }
+
+
+    //解析多模态的zip文件
+    public void resolveZipFileDatasetMoreModal(List<DatasetFilesSaveReqVO> datasetFiles){
+        List<DatasetFilesDO> insertDatasetFiles = BeanUtils.toBean(datasetFiles, DatasetFilesDO.class);
+        datasetFilesMapper.insertBatch(insertDatasetFiles, 100);
+        List<DatasetFilesDO> zipFiles = insertDatasetFiles.stream()
+                .filter(datasetFilesDO -> datasetFilesDO.getDatasetFileUrl().toLowerCase().endsWith(".zip"))
+                .collect(Collectors.toList());
+//        List<Map<>>
+        zipFiles.forEach(datasetFilesDO ->{
+            HttpURLConnection connection = DataSetReadFileUtils.readFile(datasetFilesDO.getDatasetFileUrl());
+            List<Map<String,Object>> questionAnswerList=new ArrayList<>();
+            if (connection != null) {
+                try {
+                    InputStream inputStream = connection.getInputStream();
+                    ZipProcessingResultVo zipProcessingResultVo = processZipFileStream(inputStream);
+                    String trailJson = zipProcessingResultVo.getTrailJson();
+                    JSONArray jsonArray=JSONArray.parseArray(trailJson);
+                    if(jsonArray!=null&&jsonArray.size()>0){
+                        for (int i=0;i<jsonArray.size();i++){
+                            JSONObject jsonObject = jsonArray.getJSONObject(i);
+                            JSONArray conversationsJsonArray = jsonObject.getJSONArray("conversations");
+                            JSONArray imagesJsonArray = jsonObject.getJSONArray("images");
+                            //conversationsJsonArray中奇数数据为问题，偶数数据为回答
+                            for(int j=0;j<conversationsJsonArray.size();j++){
+                                Map<String,Object> quesionanswermap=new HashMap<>();
+                                JSONObject conversationsJson=conversationsJsonArray.getJSONObject(j);
+                                if(j%2==0){
+                                    DatasetQuestionDO qdo=new DatasetQuestionDO();
+                                    qdo.setDatasetId(datasetFiles.get(0).getDatasetId());
+                                    qdo.setDatasetFilesId(datasetFilesDO.getId());
+                                    String question=conversationsJson.getString("value");
+                                    String questionFrom=conversationsJson.getString("from");
+                                    qdo.setQuestion(question);
+                                    qdo.setCreateTime(LocalDateTime.now());
+                                    qdo.setQuestionFrom(questionFrom);
+                                    qdo.setStatus(2);
+                                    datasetQuestionMapper.insert(qdo);
+                                    //获取回答数据
+                                    JSONObject conversationsAnswerJson=conversationsJsonArray.getJSONObject(j+1);
+                                    DatasetAnswerDO ado=new DatasetAnswerDO();
+                                    ado.setDatasetId(datasetFiles.get(0).getDatasetId());
+                                    ado.setDatasetFilesId(datasetFilesDO.getId());
+                                    String answer=conversationsAnswerJson.getString("value");
+                                    String answerfrom=conversationsAnswerJson.getString("from");
+                                    ado.setAnswer(answer);
+                                    ado.setAnswerFrom(answerfrom);
+                                    ado.setCreateTime(LocalDateTime.now());
+                                    ado.setQuestionId(qdo.getId());
+                                    datasetAnswerMapper.insert(ado);
+                                    quesionanswermap.put("questionid",qdo.getId());
+                                    quesionanswermap.put("answerid",ado.getId());
+                                    quesionanswermap.put("images",imagesJsonArray);
+                                    questionAnswerList.add(quesionanswermap);
+                                }
+                            }
+                        }
+                    }
+                    //将图片路径入库
+                    List<Map<String, String>> imagesList = zipProcessingResultVo.getImagesList();
+                    List<DatasetImagesDO> datasetimagesdolist=new ArrayList<>();
+                    List<DatasetQuestionAnswerImageDO> datasetQuestionAnswerImageDOList=new ArrayList<>();
+                    imagesList.forEach(url ->{
+                        DatasetImagesDO ido=new DatasetImagesDO();
+                        ido.setDatasetId(datasetFilesDO.getDatasetId());
+                        ido.setDatasetFile(datasetFilesDO.getDatasetFile());
+                        ido.setDatasetImageUrl(url.get("url").toString());
+                        ido.setCreateTime(LocalDateTime.now());
+//                        ido.setDatasetImageName()
+//                        datasetimagesdolist.add(ido);
+                        datasetImagesMapper.insert(ido);
+                        //向llm_dataset_question_answer_image表中插入数据
+                        for(int i=0;i<questionAnswerList.size();i++){
+                            Map<String, Object> map = questionAnswerList.get(i);
+                            Long questionid=Long.parseLong(map.get("questionid").toString());
+                            Long answerid=Long.parseLong(map.get("answerid").toString());
+                            JSONArray imagesJsonArray= (JSONArray) map.get("images");
+                                String imagename=url.get("imagename");
+                                for (int k=0;k<imagesJsonArray.size();k++){
+                                    if(imagesJsonArray.get(k).toString().equals(JSON_IMAGE_PERFEX+imagename)){
+                                        DatasetQuestionAnswerImageDO data=new DatasetQuestionAnswerImageDO();
+                                        data.setDatasetId(datasetFilesDO.getDatasetId());
+                                        data.setDataImageId(ido.getId());
+                                        data.setQuestionId(questionid);
+                                        data.setAnswerId(answerid);
+                                        data.setImageUrl(url.get("url").toString());
+                                        datasetQuestionAnswerImageDOList.add(data);
+                                    }
+                                }
+                        }
+                    });
+                    datasetQuestionAnswerImageMapper.insertBatch(datasetQuestionAnswerImageDOList);
+//                    datasetImagesMapper.insertBatch(datasetimagesdolist);
+
+                } catch (Exception e) {
+                    throw exception(new ErrorCode(
+                            11000, "请正确上传zip格式得数据！！！"));
+                } finally {
+                    connection.disconnect();
+                }
+            }
+        });
+    }
+
+
+
+
+    public ZipProcessingResultVo processZipFileStream(InputStream zipInputStream) throws IOException {
+        String trainJsonlContent = null;
+        List<Map<String,String>> uploadedImageUrls = new ArrayList<>();
+
+        // 使用 try-with-resources 确保 ZipInputStream 被正确关闭
+        try (ZipInputStream zis = new ZipInputStream(zipInputStream, StandardCharsets.UTF_8)) { // 假设ZIP文件条目名称是UTF-8编码
+            ZipEntry entry;
+            // 遍历ZIP文件中的每一个条目
+            while ((entry = zis.getNextEntry()) != null) {
+                String entryName = entry.getName();
+
+                // 忽略目录条目
+                if (entry.isDirectory()) {
+                    zis.closeEntry();
+                    continue;
+                }
+                String jsonfilename=entryName.split("/")[entryName.split("/").length-1];
+                // 1. 获取 train.jsonl 文件内容
+                if (DatasetMoreModalConstants.train_jsonfile.equals(jsonfilename)) {
+                    trainJsonlContent = readEntryContentAsString(zis);
+                    System.out.println("成功读取 train.jsonl 内容，大小: " + (trainJsonlContent != null ? trainJsonlContent.length() : 0) + " 字符。");
+                }
+                // 2. 处理 images 目录下的图片
+                else if (isImageFile(entryName)) {
+                    Map<String,String> map=new HashMap<>();
+                    // 提取文件名 (例如：img1.jpg)
+                    String filename = Paths.get(entryName).getFileName().toString();
+                    String contentType = getContentType(filename);
+
+                    // 获取图片大小，用于上传服务
+                    long imageSize = entry.getSize();
+
+                    // 核心：直接将zis（当前entry的流）传递给ImageUploadService
+                    // 注意：如果entry.getSize()为-1，依赖于ImageUploadService的实现
+                    // 如果ImageUploadService不能处理未知大小的流，则此处需要先将流读入内存
+                    String imageUrl;
+                    if (imageSize == -1) {
+                        System.err.println("Warning: Image entry " + entryName + " has unknown size (-1). Reading to ByteArrayOutputStream first for upload.");
+                        byte[] imageData = readEntryContentAsBytes(zis); // 读取到内存
+                        imageSize = imageData.length;
+                        InputStream tempStream = new java.io.ByteArrayInputStream(imageData);
+                        MockMultipartFile file = new MockMultipartFile("file", filename, contentType, tempStream);
+                        imageUrl = fileService.createFile(file.getOriginalFilename(), "", IoUtil.readBytes(file.getInputStream()));
+                    } else {
+                        byte[] imageData = readEntryContentAsBytes(zis);
+                        InputStream tempStream = new java.io.ByteArrayInputStream(imageData);
+                        MockMultipartFile file = new MockMultipartFile("file", filename, contentType, tempStream);
+                        // 推荐方式：直接传递zis，ImageUploadService从zis中读取entry内容
+                        imageUrl = fileService.createFile(file.getOriginalFilename(), "", IoUtil.readBytes(file.getInputStream()));
+                    }
+                    map.put("url",imageUrl);
+                    map.put("imagename",filename);
+                    uploadedImageUrls.add(map);
+                    System.out.println("成功上传图片: " + filename + "，URL: " + imageUrl);
+                }
+
+                zis.closeEntry(); // 关闭当前条目，准备读取下一个
+            }
+        } catch (IOException e) {
+            System.err.println("处理ZIP文件流失败: " + e.getMessage());
+            throw e; // 重新抛出，让控制器处理
+        }
+
+        // 检查是否找到了必要文件/图片 (可选，根据业务需求)
+//        if (trainJsonlContent == null) {
+//            System.out.println("警告: ZIP文件中未找到 " + TRAIN_JSONL_PATH);
+//            // throw new IOException(TRAIN_JSONL_PATH + " 文件缺失。");
+//        }
+//        if (uploadedImageUrls.isEmpty()) {
+//            System.out.println("警告: ZIP文件中未找到 " + IMAGES_DIR_PATH_PREFIX + " 目录下的任何图片。");
+//        }
+
+        return new ZipProcessingResultVo(trainJsonlContent, uploadedImageUrls);
+    }
+
+
+    private String readEntryContentAsString(ZipInputStream zis) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        byte[] buffer = new byte[DatasetMoreModalConstants.BUFFER_SIZE];
+        int len;
+        while ((len = zis.read(buffer)) != -1) {
+            baos.write(buffer, 0, len);
+        }
+        return baos.toString(StandardCharsets.UTF_8.name()); // 假设 train.jsonl 是 UTF-8 编码
+    }
+
+    /**
+     * 从 ZipInputStream 中读取当前条目的所有内容作为字节数组。
+     * 主要用于 ZipEntry.getSize() 为 -1 时作为备用方案，或者 ImageUploadService 不支持流式上传时。
+     */
+    private byte[] readEntryContentAsBytes(ZipInputStream zis) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        byte[] buffer = new byte[DatasetMoreModalConstants.BUFFER_SIZE];
+        int len;
+        while ((len = zis.read(buffer)) != -1) {
+            baos.write(buffer, 0, len);
+        }
+        return baos.toByteArray();
+    }
+
+    /**
+     * 判断文件名是否为常见图片格式。
+     */
+    private boolean isImageFile(String filename) {
+        String lowerCaseFilename = filename.toLowerCase();
+        return lowerCaseFilename.endsWith(".jpg") ||
+                lowerCaseFilename.endsWith(".jpeg") ||
+                lowerCaseFilename.endsWith(".png") ||
+                lowerCaseFilename.endsWith(".gif") ||
+                lowerCaseFilename.endsWith(".bmp") ||
+                lowerCaseFilename.endsWith(".webp");
+    }
+
+    /**
+     * 根据文件名后缀获取MIME Content-Type。
+     */
+    private String getContentType(String filename) {
+        Matcher matcher = IMAGE_FILE_EXTENSION_PATTERN.matcher(filename.toLowerCase());
+        if (matcher.find()) {
+            String extension = matcher.group(1);
+            switch (extension) {
+                case "jpg":
+                case "jpeg": return "image/jpeg";
+                case "png": return "image/png";
+                case "gif": return "image/gif";
+                case "bmp": return "image/bmp";
+                case "webp": return "image/webp";
+            }
+        }
+        return "application/octet-stream"; // 默认MIME类型
+    }
+
+
+
     @Override
     @Transactional
     public void updateDataset(DatasetSaveReqVO updateReqVO) {
@@ -364,8 +694,10 @@ public class DatasetServiceImpl implements DatasetService {
     public DatasetRespVO getDataset(Long id) {
         DatasetDO datasetDO = datasetMapper.selectById(id);
         DatasetRespVO datasetRespVO = BeanUtils.toBean(datasetDO, DatasetRespVO.class);
+        Integer datasetParentType = datasetDO.getDatasetParentType();
         List<DatasetFilesDO> datasetFilesDOS = datasetFilesMapper.selectList(new LambdaQueryWrapper<DatasetFilesDO>().eq(DatasetFilesDO::getDatasetId, id));
         datasetRespVO.setDatasetFiles(BeanUtils.toBean(datasetFilesDOS, DatasetFilesRespVO.class));
+
         /*List<DatasetQuestionDO> datasetQuestionDO = datasetQuestionMapper.selectList(new LambdaQueryWrapper<DatasetQuestionDO>().eq(DatasetQuestionDO::getDatasetId, id));
         List<DatasetQuestionRespVO> datasetQuestionRespVOS = BeanUtils.toBean(datasetQuestionDO, DatasetQuestionRespVO.class);
         datasetRespVO.setDatasetQuestionRespVOS(datasetQuestionRespVOS);*/
@@ -387,7 +719,41 @@ public class DatasetServiceImpl implements DatasetService {
         result.add(datasetRespVOS0);
         result.add(datasetRespVOS1);*/
         List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>()
-                .eq(DatasetDO::getStatus, 2)); // 获取所有数据集
+                .eq(DatasetDO::getStatus, 2)
+        ); // 获取所有数据集
+
+        // 创建两个根节点，分别代表两种 type
+        DatasetTreeNode privateRoot = new DatasetTreeNode(DataConstants.dataTypePrivate);
+        DatasetTreeNode publicRoot = new DatasetTreeNode(DataConstants.dataTypePublic);
+
+        for (DatasetDO datasetDO : datasetDOS) {
+            DatasetRespVO datasetRespVO = BeanUtils.toBean(datasetDO, DatasetRespVO.class);
+            // 根据 type 字段决定节点的位置
+            if (datasetRespVO.getType() == DataConstants.dataTypePrivate) {
+                privateRoot.getChildren().add(datasetRespVO);
+            } else if (datasetRespVO.getType() == DataConstants.dataTypePublic) {
+                publicRoot.getChildren().add(datasetRespVO);
+            }
+        }
+        List<DatasetTreeNode> root = new ArrayList<>();
+        root.add(privateRoot);
+        root.add(publicRoot);
+        return root;
+    }
+
+    @Override
+    public List<DatasetTreeNode> queryAllByBaseModelType(Integer type) {
+        /*List<DatasetDO> datasetDOS0 = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().eq(DatasetDO::getType, DataConstants.dataTypePrivate));
+        List<DatasetRespVO> datasetRespVOS0 = BeanUtils.toBean(datasetDOS0, DatasetRespVO.class);
+        List<DatasetDO> datasetDOS1 = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().eq(DatasetDO::getType, DataConstants.dataTypePublic));
+        List<DatasetRespVO> datasetRespVOS1 = BeanUtils.toBean(datasetDOS1, DatasetRespVO.class);
+        List<List<DatasetRespVO>> result = new ArrayList<>();
+        result.add(datasetRespVOS0);
+        result.add(datasetRespVOS1);*/
+        List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>()
+                .eq(DatasetDO::getStatus, 2)
+                .eq(DatasetDO::getDatasetParentType,type)
+        ); // 获取所有数据集
 
         // 创建两个根节点，分别代表两种 type
         DatasetTreeNode privateRoot = new DatasetTreeNode(DataConstants.dataTypePrivate);
@@ -554,7 +920,8 @@ public class DatasetServiceImpl implements DatasetService {
                     // 解析JSON数据
                     jsonParsing(content, datasetFilesDO);
                 } catch (Exception e) {
-                    throw exception(new ErrorCode(11000, "请正确上传json格式得数据！！！"));
+                    e.printStackTrace();
+//                    throw exception(new ErrorCode(11000, "请正确上传json格式得数据！！！"));
                 } finally {
                     connection.disconnect();
                 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetTaskSyncService.java

@@ -34,8 +34,17 @@ public class DatasetTaskSyncService {
         List<DatasetDO> datasetList = datasetMapper.selectList(wrapper);
         log.info("[syncDatasetAigcTask][开始同步任务，数据集数量：{}]", datasetList.size());
         datasetList.forEach(datasetDO -> {
-            List<DatasetQuestionRespVO> datasetQuestionList = datasetQuestionService.getDatasetQuestionList(datasetDO.getId());
-            dataSetService.JsonFileWrite(datasetDO,datasetQuestionList);
+            Integer datasetParentType = datasetDO.getDatasetParentType();
+            //文本数据集
+            if(datasetParentType==1){
+                List<DatasetQuestionRespVO> datasetQuestionList = datasetQuestionService.getDatasetQuestionList(datasetDO.getId());
+                dataSetService.JsonFileWrite(datasetDO,datasetQuestionList);
+            }else{
+                String dataname=datasetDO.getDatasetName();
+                //获取多模态数据集上传zip文件的路径地址
+                dataSetService.sendDatasetModal(datasetDO.getId(),dataname);
+            }
+
         });
     }
 }

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/FineTuningTaskHttpService.java

@@ -103,6 +103,42 @@ public class FineTuningTaskHttpService {
 
             // 发起 HTTP 请求
             log.debug("正在发起 HTTP POST 请求...");
+            System.out.println("---------------请求参数"+JSON.toJSONString(req));
+            String res = HttpUtils.post(requestUrl, headers, JSON.toJSONString(req));
+            log.info("HTTP 请求完成。响应内容: {}", res);
+
+            // 解析响应
+            log.debug("正在解析响应内容...");
+            AigcRespVO aigcRespVO = JSON.parseObject(res, AigcRespVO.class);
+            AigcFineTuningCreateRespVO aigcFineTuningCreateRespVO = JSON.parseObject(res, AigcFineTuningCreateRespVO.class);
+            log.info("响应解析完成。微调任务创建结果: {}", JSON.toJSONString(aigcFineTuningCreateRespVO));
+
+            // 返回结果
+            return aigcFineTuningCreateRespVO;
+
+        } catch (Exception e) {
+            log.error("创建微调任务时发生异常。请求URL: {}", requestUrl, e);
+            handleHttpException(e);
+        }
+        return null;
+    }
+
+    public AigcFineTuningCreateRespVO finetuningCreateModal (Map<String, String> headers, String url, AigcFineTuningCreateReqVO req) {
+        //  login(headers);
+        //  TODO: 在上个方法中已经将数据集的文件id赋予，调试时需要写死再放开
+        //  String fileId = "6237ed4d-a046-479c-80d6-8579a0283994";
+        //  req.setFileId(fileId);
+        String requestUrl = url + llmBackendProperties.getFinetuningCreateModal();
+        try {
+            // 记录请求信息
+            log.info("开始创建微调任务，请求URL: {}", requestUrl);
+            log.info("请求头: {}", headers);
+            log.info("请求体: {}", JSON.toJSONString(req));
+
+            // 发起 HTTP 请求
+            log.debug("正在发起 HTTP POST 请求...");
+            log.debug("请求参数为:---------------------------------"+JSON.toJSONString(req));
+            System.out.println("请求参数为:---------------------------------"+JSON.toJSONString(req));
             String res = HttpUtils.post(requestUrl, headers, JSON.toJSONString(req));
             log.info("HTTP 请求完成。响应内容: {}", res);
 

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java

@@ -777,14 +777,14 @@ public class RagHttpService {
 
         // 遍历 JSON 数组
         for (int i = 0; i < jsonArray.size(); i++) {
-            JSONArray pairArray = jsonArray.getJSONArray(i);
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
 
             // 解析文档信息
-            JSONObject documentJson = pairArray.getJSONObject(0);
-            DocumentInfoVO document = JSON.parseObject(documentJson.toJSONString(), DocumentInfoVO.class);
+//            JSONObject documentJson = jsonArray.getJSONObject(0);
+            DocumentInfoVO document = JSON.parseObject(jsonObject.toJSONString(), DocumentInfoVO.class);
 
             // 解析命中率
-            Double rate = pairArray.getDoubleValue(1);
+            Double rate = jsonObject.getDoubleValue("score");
 
             if (rate >= score) {
                 QueryResultPairVO pair = new QueryResultPairVO();
@@ -805,6 +805,61 @@ public class RagHttpService {
 
         return results;
     }
+//    private static List<QueryResultPairVO> parseHitRateTestResults (String json, Double score) {
+//        boolean array = json.trim().startsWith("[");
+//        // 先判断 JSON 是否是一个数组
+//
+//        if (!array) {
+//            // 判断是否存在 detail 字段
+//            JSONObject jsonObject = JSON.parseObject(json);
+//            if (jsonObject.containsKey("detail")) {
+//                String detail = jsonObject.getString("detail");
+//
+//                if (detail.contains("No documents found for the given query")) {
+//                    throw exception(new ErrorCode(100_100_1, "未找到符合条件的文档，请检查查询条件！"));
+//                }
+//                return new ArrayList<>();
+//            }
+//        }
+//
+//
+//        // 将 JSON 转换为 List<QueryResultPair>
+//        // 解析 JSON 数组
+//        JSONArray jsonArray = JSON.parseArray(json);
+//
+//        // 创建结果列表
+//        List<QueryResultPairVO> results = new ArrayList<>();
+//
+//        // 遍历 JSON 数组
+//        for (int i = 0; i < jsonArray.size(); i++) {
+//            JSONArray pairArray = jsonArray.getJSONArray(i);
+//
+//            // 解析文档信息
+//            JSONObject documentJson = pairArray.getJSONObject(0);
+//            DocumentInfoVO document = JSON.parseObject(documentJson.toJSONString(), DocumentInfoVO.class);
+//
+//            // 解析命中率
+//            Double rate = pairArray.getDoubleValue(1);
+//
+//            if (rate >= score) {
+//                QueryResultPairVO pair = new QueryResultPairVO();
+//                pair.setDocument(document);
+//                pair.setHitRate(rate);
+//                results.add(pair);
+//            }
+//
+//        }
+//
+//        //        // 访问数据
+//        //        for (QueryResultPairVO pair : results) {
+//        //            System.out.println("Page Content: " + pair.getDocument().getPageContent());
+//        //            System.out.println("Hit Rate: " + pair.getHitRate());
+//        //            System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId());
+//        //            System.out.println("----------------------");
+//        //        }
+//
+//        return results;
+//    }
 
     public static void main (String[] args) {
         List<String> ids = new ArrayList<>();

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/ModelCompletionsModalReqVO.java

@@ -0,0 +1,35 @@
+package cn.iocoder.yudao.module.llm.service.http.vo;
+
+import com.alibaba.fastjson.JSONArray;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@ToString
+public class ModelCompletionsModalReqVO {
+
+    private String model;
+    private List<ModelCompletionsMessageModal> messages;
+    private Integer max_tokens;
+    private Double temperature;
+    private Double top_p;
+    private Boolean stream;
+
+
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static class ModelCompletionsMessageModal {
+
+        private String role;
+        private JSONArray content;
+
+    }
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/ModelCompletionsModalRespVO.java

@@ -0,0 +1,11 @@
+package cn.iocoder.yudao.module.llm.service.http.vo;
+
+import lombok.Data;
+
+@Data
+public class ModelCompletionsModalRespVO {
+
+    private String system;
+    private String question;
+    private String answer;
+}

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/ModelCompletionsReqVO.java

@@ -1,5 +1,6 @@
 package cn.iocoder.yudao.module.llm.service.http.vo;
 
+import com.alibaba.fastjson.JSONArray;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelassesstaskmanual/ManualModelAnswerServiceImpl.java

@@ -10,6 +10,7 @@ import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.manual
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.manualAnnoVo.ManualModelAnswerSaveReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.LabelInformationVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ManualAssessmentPageRespVO;
+import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ManualEvaluationReportRespVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskDimensionRespVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskManualPageReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskManualRespVO;
@@ -81,15 +82,28 @@ public class ManualModelAnswerServiceImpl implements ManualModelAnswerService {
         PageResult<ManualModelAnswerDO> modelAnswerDOPageResult = manualModelAnswerMapper.selectPage(pageReqVO);
         List<ManualAssessmentPageRespVO> res = new ArrayList<>();
         if(!CollectionUtils.isAnyEmpty(modelAnswerDOPageResult.getList())){
-            modelAnswerDOPageResult.getList().forEach(modelAnswerDO -> {
+            // 提取模型评估人工标注的模型回答
+            List<ManualModelAnswerDO> list = modelAnswerDOPageResult.getList();
+            // 获取数据集问题
+            List<Long> collected = list.stream().map(ManualModelAnswerDO::getQuestionId).collect(Collectors.toList());
+            List<DatasetQuestionDO> datasetQuestionDOS = datasetQuestionMapper.selectBatchIds(collected);
+            Map<Long, DatasetQuestionDO> questionMap = datasetQuestionDOS.stream().collect(Collectors.toMap(DatasetQuestionDO::getId, v -> v));
+            // 获取数据集答案
+            LambdaQueryWrapper<DatasetAnswerDO> wrapper = new LambdaQueryWrapper<DatasetAnswerDO>()
+                    .in(DatasetAnswerDO::getQuestionId, collected);
+            List<DatasetAnswerDO> datasetAnswerDO = datasetAnswerMapper.selectList(wrapper);
+            Map<Long, List<DatasetAnswerDO>> collect = datasetAnswerDO.stream().collect(Collectors.groupingBy(DatasetAnswerDO::getQuestionId));
+
+            // 封装数据
+            list.forEach(modelAnswerDO -> {
                 ManualAssessmentPageRespVO resp = BeanUtils.toBean(modelAssessTaskManual, ManualAssessmentPageRespVO.class);
-                DatasetQuestionDO datasetQuestionDO = datasetQuestionMapper.selectById(modelAnswerDO.getQuestionId());
+                DatasetQuestionDO datasetQuestionDO = questionMap.get(modelAnswerDO.getQuestionId());
 
                 if (datasetQuestionDO==null){
                     throw exception(QUESTION_NOT_EXISTS);
                 }
 
-                List<DatasetAnswerDO> datasetAnswerDOS = datasetAnswerMapper.selectList(new LambdaQueryWrapper<>(DatasetAnswerDO.class).eq(DatasetAnswerDO::getQuestionId, modelAnswerDO.getQuestionId()));
+                List<DatasetAnswerDO> datasetAnswerDOS = collect.get(modelAnswerDO.getQuestionId());
                 String question = datasetQuestionDO.getQuestion();
                 if (StringUtils.isBlank(question)){
                     throw exception(QUESTION_NOT_EXISTS);

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelassesstaskmanual/ModelAssessTaskManualServiceImpl.java

@@ -71,6 +71,7 @@ public class ModelAssessTaskManualServiceImpl implements ModelAssessTaskManualSe
     @Resource
     private ManualModelAnswerMapper manualModelAnswerMapper;
 
+
     @Override
     public Long createModelAssessTaskManual (ModelAssessTaskManualSaveReqVO createReqVO) {
         validateModelAssessTaskManualNameExists(createReqVO);
@@ -126,7 +127,14 @@ public class ModelAssessTaskManualServiceImpl implements ModelAssessTaskManualSe
         wrapper.eq(ModelAssessTaskManualDO::getId, id);
         wrapper.set(ModelAssessTaskManualDO::getDeleted, true);
         // 删除
-        modelAssessTaskManualMapper.update(null, wrapper);
+//        modelAssessTaskManualMapper.update(null, wrapper);
+          modelAssessTaskManualMapper.deleteByTaskid(id);
+          //删除当前数据对应的评估维度
+        modelAssessTaskDimensionMapper.deleteDataByTaskId(id);
+        //删除答案表对应的数据
+//        LambdaUpdateWrapper<ManualModelAnswerDO> answerwrapper = new LambdaUpdateWrapper<>();
+//        answerwrapper.eq(ManualModelAnswerDO::getManalTaskId,id);
+        manualModelAnswerMapper.deleteDataByTaskId(id);
     }
 
     private void validateModelAssessTaskManualExists (Long id) {

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelassesstaskmanualbackup/ModelAssessTaskManualBackupServiceImpl.java

@@ -49,6 +49,7 @@ import org.springframework.validation.annotation.Validated;
 import javax.annotation.Resource;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.time.LocalDateTime;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -95,6 +96,7 @@ public class ModelAssessTaskManualBackupServiceImpl implements ModelAssessTaskMa
     public Long createModelAssessTaskManualBackup (ModelAssessTaskManualBackupSaveReqVO createReqVO) {
         // 插入
         ModelAssessTaskManualBackupDO modelAssessTaskManualBackup = BeanUtils.toBean(createReqVO, ModelAssessTaskManualBackupDO.class);
+        modelAssessTaskManualBackup.setCreateTime(LocalDateTime.now());
         modelAssessTaskManualBackupMapper.insert(modelAssessTaskManualBackup);
         // 返回
         return modelAssessTaskManualBackup.getId();
@@ -143,9 +145,9 @@ public class ModelAssessTaskManualBackupServiceImpl implements ModelAssessTaskMa
 
     @Override
     public void unBackup (Long id) {
-        modelAssessTaskManualBackupMapper.deleteTrue(id);
-        modelAssessTaskDimensionBackupMapper.deleteTrue(id);
-        manualModelAnswerBackupMapper.deleteTrue(id);
+        modelAssessTaskManualBackupMapper.deleteByTaskid(id);
+        modelAssessTaskDimensionBackupMapper.deleteByTaskid(id);
+        manualModelAnswerBackupMapper.deleteByTaskid(id);
     }
 
     @Override

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceService.java

@@ -92,6 +92,8 @@ public interface ModelServiceService {
 
     void startTheModel(ModelServiceSaveReqVO updateReqVO);
 
+    void startDatasetMoreModal(ModelServiceSaveReqVO updateReqVO);
+
     void disableTheModel(ModelServiceSaveReqVO updateReqVO);
 
     void inspectTheApplication(ModelServiceSaveReqVO updateReqVO);

yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/modelservice/ModelServiceServiceImpl.java

@@ -1,28 +1,23 @@
 package cn.iocoder.yudao.module.llm.service.modelservice;
 
-import cn.hutool.json.JSONUtil;
 import cn.iocoder.yudao.framework.common.exception.ErrorCode;
 import cn.iocoder.yudao.framework.common.pojo.PageResult;
 import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
 import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
-import cn.iocoder.yudao.module.llm.controller.admin.basemodel.BaseModelController;
 import cn.iocoder.yudao.module.llm.controller.admin.modelservice.vo.ModelServicePageReqVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelservice.vo.ModelServiceRespVO;
 import cn.iocoder.yudao.module.llm.controller.admin.modelservice.vo.ModelServiceSaveReqVO;
-import cn.iocoder.yudao.module.llm.dal.dataobject.application.ApplicationDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.finetuningtask.FineTuningTaskDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.modelservice.ModelServiceDO;
 import cn.iocoder.yudao.module.llm.dal.dataobject.servername.ServerNameDO;
-import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
 import cn.iocoder.yudao.module.llm.dal.mysql.finetuningtask.FineTuningTaskMapper;
 import cn.iocoder.yudao.module.llm.dal.mysql.modelservice.ModelServiceMapper;
 import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
 import cn.iocoder.yudao.module.llm.service.application.ApplicationService;
 import cn.iocoder.yudao.module.llm.service.async.AsyncModelServiceService;
 import cn.iocoder.yudao.module.llm.service.basemodel.BaseModelService;
-import cn.iocoder.yudao.module.llm.service.basemodel.BaseModelTaskService;
 import cn.iocoder.yudao.module.llm.service.http.ModelService;
 import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
 import cn.iocoder.yudao.module.llm.service.modelassesstaskauto.ModelAssessTaskAutoService;
@@ -33,8 +28,6 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.validation.annotation.Validated;
@@ -343,6 +336,7 @@ public class ModelServiceServiceImpl implements ModelServiceService {
                     item.setFineTuningTaskName(fineTuningTaskDO.getModelName());
                     item.setBaseApiUrl(fineTuningTaskDO.getApiUrl());
                     item.setBaseApiKey(fineTuningTaskDO.getApiKey());
+                    item.setModelType(fineTuningTaskDO.getModelType()==null?"":fineTuningTaskDO.getModelType());
                 }
             });
         }
@@ -474,6 +468,25 @@ public class ModelServiceServiceImpl implements ModelServiceService {
         }
     }
 
+    @Override
+    public void startDatasetMoreModal(ModelServiceSaveReqVO updateReqVO) {
+        try {
+            Long fineTuningTask = updateReqVO.getFineTuningTask();
+            BaseModelDO byAigcId = baseModelService.getById(fineTuningTask);
+            Map<String,String> map = new HashMap<>();
+            map.put("model",byAigcId.getModelName());
+            log.info("开始请求", llmBackendProperties.getDeployModel());
+            String resStr = HttpUtils.post(llmBackendProperties.getDeployModel(), null,JSON.toJSONString(map));
+            log.info(" unActive:{}", resStr);
+            ModelServiceDO updateObj = BeanUtils.toBean(updateReqVO, ModelServiceDO.class);
+            updateObj.setStatus(1);
+            updateObj.setNumber(1);
+            modelServiceMapper.updateById(updateObj);
+        }catch (Exception e){
+            log.error("启动基础模型状态时发生异常: {}", e.getMessage(), e);
+        }
+    }
+
     @Override
     public void disableTheModel(ModelServiceSaveReqVO updateReqVO) {
         try {

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetImagesMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetImagesMapper">
+
+    <!--
+        一般情况下，尽可能使用 Mapper 进行 CRUD 增删改查即可。
+        无法满足的场景，例如说多表关联查询，才使用 XML 编写 SQL。
+        代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
+        文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
+     -->
+
+</mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/dataset/DatasetQuestionAnswerImagesMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetQuestionAnswerImageMapper">
+
+    <!--
+        一般情况下，尽可能使用 Mapper 进行 CRUD 增删改查即可。
+        无法满足的场景，例如说多表关联查询，才使用 XML 编写 SQL。
+        代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
+        文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
+     -->
+
+</mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanual/ManualModelAnswerMapper.xml

@@ -8,5 +8,7 @@
         代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
         文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
      -->
-
+    <delete id="deleteDataByTaskId" parameterType="java.lang.Long">
+        delete from llm_manual_model_answer where manal_task_id=#{taskId}
+    </delete>
 </mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanual/ModelAssessTaskDimensionMapper.xml

@@ -8,5 +8,7 @@
         代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
         文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
      -->
-
+    <delete id="deleteDataByTaskId" parameterType="java.lang.Long">
+        delete from llm_model_assess_task_dimension where task_id=#{taskId}
+    </delete>
 </mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanual/ModelAssessTaskManualMapper.xml

@@ -9,4 +9,7 @@
         文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
      -->
 
+    <delete id="deleteByTaskid" parameterType="java.lang.Long">
+        delete from llm_model_assess_task_manual where id=#{taskId}
+    </delete>
 </mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanualbackup/ManualModelAnswerBackupMapper.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanualbackup.ManualModelAnswerBackupMapper">
+
+    <!--
+        一般情况下，尽可能使用 Mapper 进行 CRUD 增删改查即可。
+        无法满足的场景，例如说多表关联查询，才使用 XML 编写 SQL。
+        代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
+        文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
+     -->
+
+    <delete id="deleteByTaskid" parameterType="java.lang.Long">
+        delete from llm_manual_model_answer_backup where manal_task_id=#{taskId}
+    </delete>
+</mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanualbackup/ModelAssessTaskDimensionBackupMapper.xml

@@ -8,5 +8,7 @@
         代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
         文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
      -->
-
+    <delete id="deleteByTaskid" parameterType="java.lang.Long">
+        delete from llm_model_assess_task_dimension_backup where task_id=#{taskId}
+    </delete>
 </mapper>

yudao-module-llm/yudao-module-llm-biz/src/main/resources/mapper/modelassesstaskmanualbackup/ModelAssessTaskManualBackupMapper.xml

@@ -8,5 +8,7 @@
         代码生成器暂时只生成 Mapper XML 文件本身，更多推荐 MybatisX 快速开发插件来生成查询。
         文档可见：https://www.iocoder.cn/MyBatis/x-plugins/
      -->
-
+    <delete id="deleteByTaskid" parameterType="java.lang.Long">
+        delete from llm_model_assess_task_manual_backup where id=#{taskId}
+    </delete>
 </mapper>

yudao-server/src/main/resources/application-docker-dev.yaml

@@ -250,6 +250,8 @@ justauth:
 llm:
   backend:
     #################### 8123: RAG服务、训练集和标注相关API。 ###################
+    dataset_more_modal_api: http://221.238.217.216:4143/api/VL_files
+    finetuning_create_modal: /llm/finetuning-mm
     ### RAG服务
     #RAG健康检查 GET
     rag_health: ${RAG_Service_URL}/health

yudao-server/src/main/resources/application-local.yaml

@@ -34,7 +34,7 @@ spring:
       druid: # Druid 【连接池】相关的全局配置
         initial-size: 1 # 初始连接数
         min-idle: 1 # 最小连接池数量
-        max-active: 20 # 最大连接池数量
+        max-active: 50 # 最大连接池数量
         max-wait: 600000 # 配置获取连接等待超时的时间，单位：毫秒
         time-between-eviction-runs-millis: 60000 # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位：毫秒
         min-evictable-idle-time-millis: 300000 # 配置一个连接在池中最小生存的时间，单位：毫秒
@@ -46,21 +46,20 @@ spring:
       primary: master
       datasource:
         master:
-          url: jdbc:mysql://119.3.223.215:3306/xhllm?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true&rewriteBatchedStatements=true # MySQL Connector/J 8.X 连接的示例
-          username: xhllm_user
-          password: XNJZ-xhllm_user-!#%246
+          url: jdbc:mysql://221.238.217.216:4156/ruoyi-vue-pro?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true&rewriteBatchedStatements=true # MySQL Connector/J 8.X 连接的示例
+          username: root
+          password: 123456
         slave: # 模拟从库，可根据自己需要修改
           lazy: true # 开启懒加载，保证启动速度
-          url: jdbc:mysql://119.3.223.215:3306/xhllm?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true&nullCatalogMeansCurrent=true
-          username: xhllm_user
-          password: XNJZ-xhllm_user-!#%246
-
+          url: jdbc:mysql://221.238.217.216:4156/ruoyi-vue-pro?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true&nullCatalogMeansCurrent=true
+          username: root
+          password: 123456
   # Redis 配置。Redisson 默认的配置足够使用，一般不需要进行调优
   redis:
     host: 127.0.0.1 # 地址
     port: 6379 # 端口
     database: 0 # 数据库索引
-#    password: dev # 密码，建议生产环境开启
+    password: 123456 # 密码，建议生产环境开启
 
 --- #################### 定时任务相关配置 ####################
 
@@ -247,58 +246,60 @@ justauth:
 llm:
   backend:
     #################### 8123: RAG服务、训练集和标注相关API。 ###################
+    dataset_more_modal_api: http://221.238.217.216:4143/api/VL_files
     ### RAG服务
+    request_address: http://221.238.217.216:4143
     #RAG健康检查 GET
-    rag_health: http://36.103.199.248:8123/health
+    rag_health: http://221.238.217.216:4142/health
     #上传并向量化 POST
-    rag_embed: http://36.103.199.248:8123/embed
+    rag_embed: http://221.238.217.216:4142/embed
     #获取所有向量id GET
-    rag_ids: http://36.103.199.248:8123/ids
+    rag_ids: http://221.238.217.216:4142/ids
     #根据id获取文档 GET
-    rag_documents: http://36.103.199.248:8123/documents
+    rag_documents: http://221.238.217.216:4142/documents
     #根据id删除文档 DEL
-    rag_documents_del: http://36.103.199.248:8123/documents
+    rag_documents_del: http://221.238.217.216:4142/documents
     #根据file_id检索向量 POST
-    rag_query: http://36.103.199.248:8123/query
+    rag_query: http://221.238.217.216:4142/query
     #支持多个文件id查询向量 GET
-    rag_query_multiple: http://36.103.199.248:8123/query_multiple
+    rag_query_multiple: http://221.238.217.216:4142/query_multiple
     # 知识库向量嵌入
-    embed: http://36.103.199.248:8123/embed
+    embed: http://221.238.217.216:4142/embed
     # 知识库查询
-    embed_query: http://36.103.199.248:8123/query
+    embed_query: http://221.238.217.216:4142/query
 
     #### LLM train and service api 训练集、标注相关API
     # 训练集列表 GET
-    dataset_list: http://localhost:8123/api/mgr/datasets/list
+    dataset_list: http://221.238.217.216:4142/api/mgr/datasets/list
     # 上传训练集 POST
-    dataset_create: http://localhost:8123/api/mgr/datasets/create
+    dataset_create: http://221.238.217.216:4142/api/mgr/datasets/create
     # 删除训练集 DELETE
-    dataset_delete: http://localhost:8123/api/mgr/datasets/
+    dataset_delete: http://221.238.217.216:4142/api/mgr/datasets/
     # 训练集标注 GET
-    annotation_task_list: http://localhost:8123/api/mgr/annotation/task/list
+    annotation_task_list: http://221.238.217.216:4142/api/mgr/annotation/task/list
     # 标注信息 GET
-    annotation_task: http://localhost:8123/api/mgr/annotation/task
+    annotation_task: http://221.238.217.216:4142/api/mgr/annotation/task
     # 保存标注 POST
-    annotation_task_save: http://localhost:8123/api/mgr/annotation/task/task-6025001b-692c-44a1-9bc7-2a34bd7c0efe/segment/das-2eedd7bf-3770-4816-a961-b30c446b7a4f/mark
+    annotation_task_save: http://221.238.217.216:4142/api/mgr/annotation/task/task-6025001b-692c-44a1-9bc7-2a34bd7c0efe/segment/das-2eedd7bf-3770-4816-a961-b30c446b7a4f/mark
 
     #################### 9000: 大模型管理、微调任务、文件上传和模型部署相关API。 ###################
     # 大模型列表 GET
-    models_list: http://36.103.199.248:9000/api/models
+    models_list: http://221.238.217.216:9000/api/models
     # 登录 POST
-    login: http://36.103.199.248:9000/api/auth/login
-    account: http://36.103.199.248:9000/api/auth/account
+    login: http://221.238.217.216:9000/api/auth/login
+    account: http://221.238.217.216:9000/api/auth/account
     login_username: admin
     login_password: admin
     # 微调任务详情 GET
-    finetuning_detail: http://36.103.199.248:9000/api/finetuning
+    finetuning_detail: http://221.238.217.216:9000/api/finetuning
     # 微调任务取消
-    finetuning_cancel: http://36.103.199.248:9000/api/finetuning/%s/cancel
+    finetuning_cancel: http://221.238.217.216:9000/api/finetuning/%s/cancel
     # 微调文件列表 GET
-    finetuning_file_list: http://36.103.199.248:9000/api/files?purpose=fine-tune
+    finetuning_file_list: http://221.238.217.216:9000/api/files?purpose=fine-tune
     # 模型部署
-    model_create: http://36.103.199.248:9000/api/models
+    model_create: http://221.238.217.216:9000/api/models
     # aigc模型推理
-    aigc_model_completions: http://36.103.199.248:9000/api/channels/chat/completions
+    aigc_model_completions: http://221.238.217.216:9000/api/channels/chat/completions
 
 
     #################### 5123: 微调任务、模型部署、文件管理、提示词优化、自动评估、文生图等API。 ###################
@@ -306,6 +307,7 @@ llm:
     # 微调文件上传
     aigc_file_upload: /api/files
     finetuning_create: /llm/finetuning
+    finetuning_create_modal: /llm/finetuning-mm
     # 日志获取
     finetuning_log: /llm/get_log
     # 开始部署
@@ -319,41 +321,41 @@ llm:
     # 模型文件下载
     model_file_download: /models/download/?file_path=
     # 提示词优化
-    optimize_prompt: http://36.103.199.248:5123/optimize-prompt
+    optimize_prompt: http://221.238.217.216:4143/optimize-prompt
     # 自动评估
     auto_evaluation: /llm-eval
     # 文生图
-    text_to_image: http://36.103.199.248:5123/generate-image
+    text_to_image: http://221.238.217.216:4143/generate-image
     # 检查点文件列表
     check_file_list: /llm/finetuning/checkpoints?model_name=
     # 模型调优停止 POST
     stop_finetuning: /llm/finetuning/stop
     # 基座模型状态 POST
-    base_model_status: http://36.103.199.248:5123/llm/deploy/list
+    base_model_status: http://221.238.217.216:4143/llm/deploy/list
     # 模型部署 POST
-    deploy_model: http://36.103.199.248:5123/llm/deploy
+    deploy_model: http://221.238.217.216:4143/llm/deploy
     # 模型删除
-    delete_model: http://36.103.199.248:5123/llm/deploy/stop
+    delete_model: http://221.238.217.216:4143/llm/deploy/stop
     # 模型列表 get
-    a_list_of_available_models: http://36.103.199.248:5123/llm/list
+    a_list_of_available_models: http://221.238.217.216:4143/llm/list
     #删除模型
     delete_the_model: /delete_model
-    delete_the_model_full: http://36.103.199.248:5123/delete_model
+    delete_the_model_full: http://221.238.217.216:4143/delete_model
     # 知识库分块文件
-    knowledgeBaseChunkedFiles : http:/36.103.199.248:8123/documents
+    knowledgeBaseChunkedFiles : http:/221.238.217.216:4142/documents
 
 
     #################### 30000: 大模型对话相关API。 ###################
     #### 大模型对话
     # 模型列表 GET
-    base_model_list: http://36.103.199.248:30000/model/v1/models
+    base_model_list: http://221.238.217.216:30000/model/v1/models
     # 模型对话 POST
-    model_completions: http://36.103.199.248:30000/v1/chat/completions
+    model_completions: http://221.238.217.216:30000/v1/chat/completions
 
     #################### 48080: 应用和管理服务相关API。 ###################
-    application_api: http://localhost:48080/admin-api/llm/application/api/apiKey/chat
+    application_api: http://221.238.217.216:48080/admin-api/llm/application/api/apiKey/chat
 
-    model_service_api: http://localhost:48080/admin-api/llm/model-service/api/apiKey/chat
+    model_service_api: http://221.238.217.216:48080/admin-api/llm/model-service/api/apiKey/chat
 
 
 --- #################### iot相关配置 TODO 芋艿：再瞅瞅 ####################

yudao-server/src/main/resources/application-prod.yaml

@@ -247,6 +247,8 @@ justauth:
 llm:
   backend:
     #################### 8123: RAG服务、训练集和标注相关API。 ###################
+    dataset_more_modal_api: http://221.238.217.216:4143/api/VL_files
+    finetuning_create_modal: /llm/finetuning-mm
     ### RAG服务
     #RAG健康检查 GET
     rag_health: http://127.0.0.1:8123/health

yudao-server/src/main/resources/application-ymx.yaml

@@ -247,6 +247,8 @@ justauth:
 llm:
   backend:
     #################### 8123: RAG服务、训练集和标注相关API。 ###################
+    dataset_more_modal_api: http://221.238.217.216:4143/api/VL_files
+    finetuning_create_modal: /llm/finetuning-mm
     ### RAG服务
     #RAG健康检查 GET
     rag_health: http://127.0.0.1:8123/health