From d5c789bea1cc151ed95d3cd7b957f09ee8c29741 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 10:14:21 +0800 Subject: [PATCH 1/6] =?UTF-8?q?fix(llm):=20=E4=BF=AE=E5=A4=8D=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E7=9F=A5=E8=AF=86=E5=BA=93=E6=97=B6=E6=9C=AA=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=AF=B9=E5=BA=94=E6=96=87=E6=A1=A3=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98-=20=E6=9B=B4=E6=96=B0=20KnowledgeBaseServiceImpl=20?= =?UTF-8?q?=E4=B8=AD=E5=88=A0=E9=99=A4=E7=9F=A5=E8=AF=86=E5=BA=93=E7=9A=84?= =?UTF-8?q?=E9=80=BB=E8=BE=91=20-=20=E4=BF=AE=E6=94=B9=20KnowledgeDocument?= =?UTF-8?q?sMapper=20=E4=B8=AD=E8=8E=B7=E5=8F=96=E5=B7=B2=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=96=87=E6=A1=A3=20ID=20=E7=9A=84=20SQL=20=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=20-=20=E4=BF=AE=E6=AD=A3=20RagHttpService=20=E4=B8=AD?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=9F=A5=E8=AF=86=E6=96=87=E6=A1=A3=E7=9A=84?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/knowledgedocuments/KnowledgeDocumentsMapper.java | 4 ++-- .../yudao/module/llm/service/http/RagHttpService.java | 2 +- .../llm/service/knowledgebase/KnowledgeBaseServiceImpl.java | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java index 78367b746..5527be93d 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java @@ -28,7 +28,7 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX selectDeleteIds(Long id); -} \ No newline at end of file +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java index e34a2cb7e..4ef269755 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java @@ -723,7 +723,7 @@ public class RagHttpService { private KnowledgeDocumentsDO getKnowledgeDocuments (Long id, String fileId) { return knowledgeDocumentsMapper.selectOne(new LambdaQueryWrapperX() .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, id) - .eq(KnowledgeDocumentsDO::getId, fileId)); + .eq(KnowledgeDocumentsDO::getFileId, fileId)); } /** diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index db3cf7fda..1131a78ff 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -120,6 +120,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { return; } + // 获取需要保留的文档 ID List retainedIds = documents.stream() .map(KnowledgeDocumentsSaveReqVO::getId) @@ -186,9 +187,9 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); docDO.setKnowledgeBaseId(knowledgeBaseId); if (doc.getId() == null) { - newDocuments.add(docDO); // 收集新增文档 + newDocuments.add(docDO); } - knowledgeDocumentsMapper.insertOrUpdate(docDO); // 更新或插入文档 + knowledgeDocumentsMapper.insertOrUpdate(docDO); }); return newDocuments; } From 0a79f153efcf3e95c64d79ccea0c8379c4b4a15f Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 11:02:13 +0800 Subject: [PATCH 2/6] =?UTF-8?q?refactor(module-llm):=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E6=96=87=E6=A1=A3=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除了对文档 ID 的非空过滤，因为 KnowledgeDocumentsSaveReqVO 已确保 ID 不为 null --- .../llm/service/knowledgebase/KnowledgeBaseServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 1131a78ff..a5f72afbe 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -124,7 +124,6 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { // 获取需要保留的文档 ID List retainedIds = documents.stream() .map(KnowledgeDocumentsSaveReqVO::getId) - .filter(Objects::nonNull) .collect(Collectors.toList()); // 删除不需要保留的文档 From 31a13cdfd0a0785297aeccab1574500be10f9363 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 11:20:20 +0800 Subject: [PATCH 3/6] =?UTF-8?q?refactor(llm):=20=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E6=8E=89=E6=A8=A1=E5=9E=8B=E9=87=8D=E6=96=B0=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 BaseModelTaskService 类中，注释掉了处理模型状态为 "stop" 时的重新部署逻辑 - 保留了原有的日志输出和错误处理逻辑 - 此修改可能是为了暂时禁用模型自动重新部署的功能，或者准备替换为新的部署策略 --- .../basemodel/BaseModelTaskService.java | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java index 182a7dcc5..9ffffccee 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java @@ -99,33 +99,33 @@ public class BaseModelTaskService { String status = latestRecord.getStatus(); log.info("最新模型部署记录状态: {}", status); - // 如果模型状态为 "stop"，则重新部署 - if ("stop".equals(status)) { - log.info("模型状态为 'stop'，正在重新部署模型..."); - - // 构建模型部署请求 - AigcModelDeploySaveReq aigcModelDeploySaveReq = new AigcModelDeploySaveReq( - baseModelDO.getAigcModelName(), "gpu"); - log.info("模型部署请求参数: {}", JSON.toJSONString(aigcModelDeploySaveReq)); - - // 发起模型部署请求 - ModelDeployRespVO modelDeployRespVO = trainHttpService.modelDeploy( - new HashMap<>(), serverName.getHost(), aigcModelDeploySaveReq); - log.info("模型部署请求完成。响应内容: {}", JSON.toJSONString(modelDeployRespVO)); - - // 更新基础模型信息 - if (!"error".equals(modelDeployRespVO.getMessage())) { - log.info("模型部署成功。正在更新基础模型信息..."); - BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO(); - baseModelSaveReqVO.setId(baseModelDO.getId()); - baseModelSaveReqVO.setModelId(modelDeployRespVO.getId()); - baseModelSaveReqVO.setChatUrl(modelDeployRespVO.getPort() + DEFAULT_MODEL_URL_SUFFIX); - baseModelService.updateBaseModel(baseModelSaveReqVO); - log.info("基础模型信息更新完成。模型ID: {}", baseModelDO.getId()); - } else { - log.error("模型部署失败。模型ID: {}", baseModelDO.getId()); - } - } +// // 如果模型状态为 "stop"，则重新部署 +// if ("stop".equals(status)) { +// log.info("模型状态为 'stop'，正在重新部署模型..."); +// +// // 构建模型部署请求 +// AigcModelDeploySaveReq aigcModelDeploySaveReq = new AigcModelDeploySaveReq( +// baseModelDO.getAigcModelName(), "gpu"); +// log.info("模型部署请求参数: {}", JSON.toJSONString(aigcModelDeploySaveReq)); +// +// // 发起模型部署请求 +// ModelDeployRespVO modelDeployRespVO = trainHttpService.modelDeploy( +// new HashMap<>(), serverName.getHost(), aigcModelDeploySaveReq); +// log.info("模型部署请求完成。响应内容: {}", JSON.toJSONString(modelDeployRespVO)); +// +// // 更新基础模型信息 +// if (!"error".equals(modelDeployRespVO.getMessage())) { +// log.info("模型部署成功。正在更新基础模型信息..."); +// BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO(); +// baseModelSaveReqVO.setId(baseModelDO.getId()); +// baseModelSaveReqVO.setModelId(modelDeployRespVO.getId()); +// baseModelSaveReqVO.setChatUrl(modelDeployRespVO.getPort() + DEFAULT_MODEL_URL_SUFFIX); +// baseModelService.updateBaseModel(baseModelSaveReqVO); +// log.info("基础模型信息更新完成。模型ID: {}", baseModelDO.getId()); +// } else { +// log.error("模型部署失败。模型ID: {}", baseModelDO.getId()); +// } +// } } else { log.warn("未找到模型部署记录。模型ID: {}", modelId); } From 18f4aeeda1d92a93f90eb3cab25cd6643236410e Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 14:21:26 +0800 Subject: [PATCH 4/6] =?UTF-8?q?refactor(llm):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E6=9B=B4=E6=96=B0=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=E5=B9=B6=E6=B7=BB=E5=8A=A0=20Embedding=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 KnowledgeEmbeddingReqVO 类用于知识库 Embedding 请求参数 - 在 KnowledgeBaseController 中添加 embeddingKnowledge 方法处理 Embedding 请求 - 在 KnowledgeBaseService接口中新增 embeddingKnowledge 方法 - 在 KnowledgeBaseServiceImpl 中实现 embeddingKnowledge 方法，重构知识库更新流程- 修改 AsyncKnowledgeBase 类，分离创建和删除知识库的方法 - 更新 KnowledgeDocumentsMapper，调整删除文档的查询逻辑 - 移除 KnowledgeDocumentsSaveReqVO 中的冗余注释 --- .../KnowledgeBaseController.java | 15 +++ .../vo/KnowledgeEmbeddingReqVO.java | 30 +++++ .../vo/KnowledgeHitRateTestReqVO.java | 2 +- .../vo/KnowledgeDocumentsSaveReqVO.java | 3 +- .../KnowledgeDocumentsMapper.java | 3 +- .../llm/service/async/AsyncKnowledgeBase.java | 45 +++---- .../knowledgebase/KnowledgeBaseService.java | 12 ++ .../KnowledgeBaseServiceImpl.java | 112 ++++++++++-------- 8 files changed, 148 insertions(+), 74 deletions(-) create mode 100644 yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java index 2098c82a4..fc3c25cb3 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java @@ -55,6 +55,14 @@ public class KnowledgeBaseController { return success(result); } + @PutMapping("/updateHitParam") + @Operation(summary = "更新知识库命中测试参数") + public CommonResult updateHitParam(@Valid @RequestBody KnowledgeBaseSaveReqVO updateReqVO) { + knowledgeBaseService.updateHitParam(updateReqVO); + return success(true); + } + + @PutMapping("/update") @Operation(summary = "更新知识库") // @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')") @@ -63,6 +71,13 @@ public class KnowledgeBaseController { return success(true); } + @PutMapping("/embeddingKnowledge") + @Operation(summary = "更新 Embedding", description = "对文件内容进行 Embedding 处理并更新到知识库") + public CommonResult embeddingKnowledge(@Valid @RequestBody KnowledgeEmbeddingReqVO request) { + knowledgeBaseService.embeddingKnowledge(request); + return success(true); + } + @PutMapping("/updateKnowledgeBaseInfo") @Operation(summary = "更新知识库") // @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')") diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java new file mode 100644 index 000000000..76353461e --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java @@ -0,0 +1,30 @@ +package cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo; + +import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsSaveReqVO; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +/** + * @Description 知识库 Embedding 请求参数 + * @Date 2025/3/14 13:05 + */ +@Schema(description = "知识库 Embedding 请求参数") +@Data +public class KnowledgeEmbeddingReqVO { + @Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "25984") + private Long id; + + @Schema(description = "知识内容长度", example = "1000") + private Integer contentLength; + + @Schema(description = "分块大小（单位：字符）", example = "512") + private Integer chunkSize; + + @Schema(description = "分块重叠大小（单位：字符）", example = "128") + private Integer chunkOverlap; + + @Schema(description = "关联的知识文档列表") + private List documents; +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java index 5b23ee7b6..d221ecda5 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java @@ -25,7 +25,7 @@ public class KnowledgeHitRateTestReqVO { * 返回结果的条数（k值） */ // @NotNull(message = "k值不能为空") - private Integer k; + private Integer topK; /** * Score阈值 diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java index 74c2a7a4d..b0ad2918f 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java @@ -13,7 +13,6 @@ public class KnowledgeDocumentsSaveReqVO { private Long id; @Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "18229") -// @NotNull(message = "知识库ID不能为空") private Long knowledgeBaseId; @Schema(description = "文档名称", example = "芋艿") @@ -28,4 +27,4 @@ public class KnowledgeDocumentsSaveReqVO { @Schema(description = "文件id", example = "1") private Long fileId; -} \ No newline at end of file +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java index 5527be93d..906f9269e 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java @@ -28,7 +28,6 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX selectDeleteIds(Long id); - } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 5fb33ca41..3de48d8b7 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -10,9 +10,9 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.RagHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; -import cn.iocoder.yudao.module.llm.service.http.vo.RegUploadReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; +import com.alibaba.fastjson.JSON; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +23,6 @@ import java.io.IOException; import java.math.RoundingMode; import java.text.DecimalFormat; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -44,24 +43,12 @@ public class AsyncKnowledgeBase { private KnowledgeDocumentsMapper knowledgeDocumentsMapper; - // 向向量知识库创建文件 - // @Async - public void createKnowledgeBase (List knowledgeList, List ids, Map knowledgeParameters) { - log.info("开始执行 createKnowledgeBase 方法。knowledgeList 大小: {}, ids 大小: {}", knowledgeList.size(), ids.size()); - - // 如果提供了 ids，则删除现有的知识库文档 - if (!CollectionUtils.isAnyEmpty(ids)) { - log.info("正在删除现有的知识库文档，ids: {}", ids); - try { - String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids); - log.info("删除知识库信息: {}", mes); - } catch (Exception e) { - log.error("删除知识库信息失败: {}", e.getMessage()); - throw exception(new ErrorCode(10047, "删除知识库信息失败！")); - } - } else { - log.info("未提供 ids，跳过删除操作。"); - } + /** + * 向向量知识库创建文件 + * @param knowledgeList + * @param knowledgeParameters + */ + public void createKnowledgeBase (List knowledgeList, Map knowledgeParameters) { // 处理 knowledgeList 中的每个知识文档 if (!CollectionUtils.isAnyEmpty(knowledgeList)) { @@ -100,6 +87,24 @@ public class AsyncKnowledgeBase { log.info("createKnowledgeBase 方法执行完成。"); } + public void deletedKnowledgeBase ( List ids) { + log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); + + // 如果提供了 ids，则删除现有的知识库文档 + if (!CollectionUtils.isAnyEmpty(ids)) { + log.info("正在删除现有的知识库文档，ids: {}", ids); + try { + String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids); + log.info("删除知识库信息: {}", mes); + } catch (Exception e) { + log.error("删除知识库信息失败: {}", e.getMessage()); + throw exception(new ErrorCode(10047, "删除知识库信息失败！")); + } + } else { + log.info("未提供 ids，跳过删除操作。"); + } + + } /** * 修改知识库文档状态 * diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java index 35653d8b3..5bd31dfce 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java @@ -75,4 +75,16 @@ public interface KnowledgeBaseService { * @return 返回结果 */ List executeHitRateTest (@Valid KnowledgeHitRateTestReqVO testReqVO); + + /** + * 知识库Embedding + * @param request + */ + void embeddingKnowledge (@Valid KnowledgeEmbeddingReqVO request); + + /** + * 更新知识库命中测试参数 + * @param updateReqVO + */ + void updateHitParam (@Valid KnowledgeBaseSaveReqVO updateReqVO); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index a5f72afbe..0a5ec2a24 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -60,31 +60,28 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { @Override @Transactional(rollbackFor = Exception.class) public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) { - // 1. 校验知识库是否存在 - validateKnowledgeParam(updateReqVO); - - // 2. 更新知识库主表基础信息 - KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); - knowledgeBaseMapper.updateById(updateObj); - - // 3. 处理附表（知识文档）数据 - handleKnowledgeDocuments(updateReqVO, updateObj); + // // 1. 校验知识库是否存在 + // validateKnowledgeParam(updateReqVO); + // + // // 2. 更新知识库主表基础信息 + // KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); + // knowledgeBaseMapper.updateById(updateObj); + // + // // 3. 处理附表（知识文档）数据 + // handleKnowledgeDocuments(updateReqVO, updateObj); } /** * 校验知识库参数 * - * @param updateReqVO 更新知识库参数 + * @param request 更新知识库参数 */ - private void validateKnowledgeParam (KnowledgeBaseSaveReqVO updateReqVO) { + private void validateKnowledgeParam (KnowledgeEmbeddingReqVO request) { // 1. 校验知识库是否存在 - validateKnowledgeBaseExists(updateReqVO.getId()); + validateKnowledgeBaseExists(request.getId()); - // 2. 校验知识库名称是否重复 - validateKnowledgeBaseNameExists(updateReqVO); - - // 3. 校验分块大小和分块重叠是否正确 - validateChunkParameters(updateReqVO.getChunkSize(), updateReqVO.getChunkOverlap()); + // 2. 校验分块大小和分块重叠是否正确 + validateChunkParameters(request.getChunkSize(), request.getChunkOverlap()); } /** @@ -109,36 +106,21 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { /** * 处理知识文档数据 * - * @param updateReqVO 更新知识库参数 - * @param updateObj 更新知识库对象 + * @param request 更新知识库参数 + * @param updateObj 更新知识库对象 */ - private void handleKnowledgeDocuments (KnowledgeBaseSaveReqVO updateReqVO, KnowledgeBaseDO updateObj) { - List documents = updateReqVO.getKnowledgeDocuments(); - if (CollectionUtils.isAnyEmpty(documents)) { - // 如果传入的文档列表为空，则删除所有关联文档 - deleteAllDocuments(updateReqVO.getId()); - return; - } + private void handleKnowledgeDocuments (KnowledgeEmbeddingReqVO request, KnowledgeBaseDO updateObj) { + // 1: 删除所有旧文档 + deleteAllDocuments(updateObj.getId()); - - // 获取需要保留的文档 ID - List retainedIds = documents.stream() - .map(KnowledgeDocumentsSaveReqVO::getId) - .collect(Collectors.toList()); - - // 删除不需要保留的文档 - deleteUnretainedDocuments(updateReqVO.getId(), retainedIds); - - // 更新或插入文档数据 - List newDocuments = updateOrInsertDocuments(documents, updateReqVO.getId(), updateObj.getKnowledgeLength()); + // 2: 重新上传文档 + List newDocuments = updateOrInsertDocuments(request.getDocuments(), request.getId()); Map knowledgeParameters = new HashMap<>(); - knowledgeParameters.put("chunkSize", updateReqVO.getChunkSize()); - knowledgeParameters.put("chunkOverlap", updateReqVO.getChunkOverlap()); + knowledgeParameters.put("chunkSize", request.getChunkSize()); + knowledgeParameters.put("chunkOverlap", request.getChunkOverlap()); - // 异步处理新增文档和删除的文档 - List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); - asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds, knowledgeParameters); + asyncKnowledgeBase.createKnowledgeBase(newDocuments, knowledgeParameters); } /** @@ -153,7 +135,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { // 异步处理删除的文档 List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(knowledgeBaseId); if (!CollectionUtils.isAnyEmpty(deleteIds)) { - asyncKnowledgeBase.createKnowledgeBase(new ArrayList<>(), deleteIds, new HashMap<>()); + asyncKnowledgeBase.deletedKnowledgeBase(deleteIds); } } @@ -177,10 +159,9 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { * * @param documents 需要更新的文档数据 * @param knowledgeBaseId 知识库 ID - * @param chunkSize * @return 更新或插入的文档数据 */ - private List updateOrInsertDocuments (List documents, Long knowledgeBaseId, Integer chunkSize) { + private List updateOrInsertDocuments (List documents, Long knowledgeBaseId) { List newDocuments = new ArrayList<>(); documents.forEach(doc -> { KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); @@ -315,17 +296,17 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { } Integer topK = 4; if (baseDO.getTopK() == null || baseDO.getTopK() <= 0) { - testReqVO.setK(topK); + testReqVO.setTopK(topK); } else { - topK = baseDO.getTopK(); - testReqVO.setK(topK); + topK = testReqVO.getTopK(); + testReqVO.setTopK(topK); } Double score = 0.2; if (baseDO.getScore() == null || baseDO.getTopK() <= 0.0 || baseDO.getScore() > 1) { testReqVO.setScore(score); } else { - score = baseDO.getScore(); + score = testReqVO.getScore(); testReqVO.setScore(score); } @@ -349,6 +330,39 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { return result; } + /** + * 知识库Embedding + * @param request + */ + @Override + public void embeddingKnowledge (KnowledgeEmbeddingReqVO request) { + // 1. 校验知识库是否存在 + validateKnowledgeParam(request); + + // 2. 更新知识库主表基础信息 + KnowledgeBaseDO updateObj = BeanUtils.toBean(request, KnowledgeBaseDO.class); + knowledgeBaseMapper.updateById(updateObj); + + // 3. 处理附表（知识文档）数据 + handleKnowledgeDocuments(request, updateObj); + } + + /** + * 更新知识库命中测试参数 + * @param updateReqVO + */ + @Override + public void updateHitParam (KnowledgeBaseSaveReqVO updateReqVO) { + // 校验存在 + validateKnowledgeBaseExists(updateReqVO.getId()); + // 更新 + KnowledgeBaseDO updateObj = new KnowledgeBaseDO(); + updateObj.setId(updateReqVO.getId()) + .setTopK(updateReqVO.getTopK()) + .setScore(updateReqVO.getScore()) + ; + knowledgeBaseMapper.updateById(updateObj); + } /** * 校验知识库是否存在 From 5f49c0094c0eeebe28ec72e1f28450d4508552a2 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 14:22:19 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix(llm):=20=E4=BF=AE=E5=A4=8D=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E6=9F=A5=E8=AF=A2=E7=9B=B8=E5=85=B3=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将 QueryMultipleReqVO 对象中的 setK 方法调用改为 setTopK --- .../yudao/module/llm/service/async/AsyncKnowledgeBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 3de48d8b7..74b8fe20e 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -148,7 +148,7 @@ public class AsyncKnowledgeBase { QueryMultipleReqVO vo = new QueryMultipleReqVO(); vo.setQuery(testReqVO.getQuery()); vo.setFileIds(fileIdStr); - vo.setK(testReqVO.getK()); + vo.setK(testReqVO.getTopK()); vo.setScore(testReqVO.getScore()); List resultList = new ArrayList<>(); From 63ef71fab3f952d89bdfc7d0fd992f5a4492ea65 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Fri, 14 Mar 2025 15:23:57 +0800 Subject: [PATCH 6/6] =?UTF-8?q?refactor(llm):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E6=9C=8D=E5=8A=A1=E5=92=8C=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修改 KnowledgeBaseServiceImpl 中的文档更新逻辑 - 更新 KnowledgeDocumentsMapper 中的 SQL 查询 - 调整 KnowledgeEmbeddingReqVO 中的字段名称 --- .../knowledgebase/vo/KnowledgeEmbeddingReqVO.java | 2 +- .../KnowledgeDocumentsMapper.java | 2 +- .../knowledgebase/KnowledgeBaseServiceImpl.java | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java index 76353461e..8ed36d5fd 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java @@ -26,5 +26,5 @@ public class KnowledgeEmbeddingReqVO { private Integer chunkOverlap; @Schema(description = "关联的知识文档列表") - private List documents; + private List knowledgeDocuments; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java index 906f9269e..8ddde8160 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java @@ -28,6 +28,6 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX selectDeleteIds(Long id); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 0a5ec2a24..916b5cd3c 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -114,7 +114,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { deleteAllDocuments(updateObj.getId()); // 2: 重新上传文档 - List newDocuments = updateOrInsertDocuments(request.getDocuments(), request.getId()); + List newDocuments = updateOrInsertDocuments(request.getKnowledgeDocuments(), request.getId()); Map knowledgeParameters = new HashMap<>(); knowledgeParameters.put("chunkSize", request.getChunkSize()); @@ -166,9 +166,10 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { documents.forEach(doc -> { KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); docDO.setKnowledgeBaseId(knowledgeBaseId); - if (doc.getId() == null) { +// if (doc.getId() == null) { newDocuments.add(docDO); - } +// } + docDO.setId(null); knowledgeDocumentsMapper.insertOrUpdate(docDO); }); return newDocuments; @@ -297,9 +298,15 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { Integer topK = 4; if (baseDO.getTopK() == null || baseDO.getTopK() <= 0) { testReqVO.setTopK(topK); + if (testReqVO.getTopK() == null){ + testReqVO.setTopK(baseDO.getTopK()); + } } else { topK = testReqVO.getTopK(); testReqVO.setTopK(topK); + if (testReqVO.getScore() == null){ + testReqVO.setScore(baseDO.getScore()); + } } Double score = 0.2;