diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java index 2098c82a4..fc3c25cb3 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java @@ -55,6 +55,14 @@ public class KnowledgeBaseController { return success(result); } + @PutMapping("/updateHitParam") + @Operation(summary = "更新知识库命中测试参数") + public CommonResult updateHitParam(@Valid @RequestBody KnowledgeBaseSaveReqVO updateReqVO) { + knowledgeBaseService.updateHitParam(updateReqVO); + return success(true); + } + + @PutMapping("/update") @Operation(summary = "更新知识库") // @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')") @@ -63,6 +71,13 @@ public class KnowledgeBaseController { return success(true); } + @PutMapping("/embeddingKnowledge") + @Operation(summary = "更新 Embedding", description = "对文件内容进行 Embedding 处理并更新到知识库") + public CommonResult embeddingKnowledge(@Valid @RequestBody KnowledgeEmbeddingReqVO request) { + knowledgeBaseService.embeddingKnowledge(request); + return success(true); + } + @PutMapping("/updateKnowledgeBaseInfo") @Operation(summary = "更新知识库") // @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')") diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java new file mode 100644 index 000000000..76353461e --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java @@ -0,0 +1,30 @@ +package cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo; + +import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsSaveReqVO; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +/** + * @Description 知识库 Embedding 请求参数 + * @Date 2025/3/14 13:05 + */ +@Schema(description = "知识库 Embedding 请求参数") +@Data +public class KnowledgeEmbeddingReqVO { + @Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "25984") + private Long id; + + @Schema(description = "知识内容长度", example = "1000") + private Integer contentLength; + + @Schema(description = "分块大小(单位:字符)", example = "512") + private Integer chunkSize; + + @Schema(description = "分块重叠大小(单位:字符)", example = "128") + private Integer chunkOverlap; + + @Schema(description = "关联的知识文档列表") + private List documents; +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java index 5b23ee7b6..d221ecda5 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java @@ -25,7 +25,7 @@ public class KnowledgeHitRateTestReqVO { * 返回结果的条数(k值) */ // @NotNull(message = "k值不能为空") - private Integer k; + private Integer topK; /** * Score阈值 diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java index 74c2a7a4d..b0ad2918f 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java @@ -13,7 +13,6 @@ public class KnowledgeDocumentsSaveReqVO { private Long id; @Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "18229") -// @NotNull(message = "知识库ID不能为空") private Long knowledgeBaseId; @Schema(description = "文档名称", example = "芋艿") @@ -28,4 +27,4 @@ public class KnowledgeDocumentsSaveReqVO { @Schema(description = "文件id", example = "1") private Long fileId; -} \ No newline at end of file +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java index 5527be93d..906f9269e 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java @@ -28,7 +28,6 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX selectDeleteIds(Long id); - } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 5fb33ca41..3de48d8b7 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -10,9 +10,9 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.RagHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; -import cn.iocoder.yudao.module.llm.service.http.vo.RegUploadReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; +import com.alibaba.fastjson.JSON; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +23,6 @@ import java.io.IOException; import java.math.RoundingMode; import java.text.DecimalFormat; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -44,24 +43,12 @@ public class AsyncKnowledgeBase { private KnowledgeDocumentsMapper knowledgeDocumentsMapper; - // 向向量知识库创建文件 - // @Async - public void createKnowledgeBase (List knowledgeList, List ids, Map knowledgeParameters) { - log.info("开始执行 createKnowledgeBase 方法。knowledgeList 大小: {}, ids 大小: {}", knowledgeList.size(), ids.size()); - - // 如果提供了 ids,则删除现有的知识库文档 - if (!CollectionUtils.isAnyEmpty(ids)) { - log.info("正在删除现有的知识库文档,ids: {}", ids); - try { - String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids); - log.info("删除知识库信息: {}", mes); - } catch (Exception e) { - log.error("删除知识库信息失败: {}", e.getMessage()); - throw exception(new ErrorCode(10047, "删除知识库信息失败!")); - } - } else { - log.info("未提供 ids,跳过删除操作。"); - } + /** + * 向向量知识库创建文件 + * @param knowledgeList + * @param knowledgeParameters + */ + public void createKnowledgeBase (List knowledgeList, Map knowledgeParameters) { // 处理 knowledgeList 中的每个知识文档 if (!CollectionUtils.isAnyEmpty(knowledgeList)) { @@ -100,6 +87,24 @@ public class AsyncKnowledgeBase { log.info("createKnowledgeBase 方法执行完成。"); } + public void deletedKnowledgeBase ( List ids) { + log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); + + // 如果提供了 ids,则删除现有的知识库文档 + if (!CollectionUtils.isAnyEmpty(ids)) { + log.info("正在删除现有的知识库文档,ids: {}", ids); + try { + String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids); + log.info("删除知识库信息: {}", mes); + } catch (Exception e) { + log.error("删除知识库信息失败: {}", e.getMessage()); + throw exception(new ErrorCode(10047, "删除知识库信息失败!")); + } + } else { + log.info("未提供 ids,跳过删除操作。"); + } + + } /** * 修改知识库文档状态 * diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java index 35653d8b3..5bd31dfce 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java @@ -75,4 +75,16 @@ public interface KnowledgeBaseService { * @return 返回结果 */ List executeHitRateTest (@Valid KnowledgeHitRateTestReqVO testReqVO); + + /** + * 知识库Embedding + * @param request + */ + void embeddingKnowledge (@Valid KnowledgeEmbeddingReqVO request); + + /** + * 更新知识库命中测试参数 + * @param updateReqVO + */ + void updateHitParam (@Valid KnowledgeBaseSaveReqVO updateReqVO); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index a5f72afbe..0a5ec2a24 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -60,31 +60,28 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { @Override @Transactional(rollbackFor = Exception.class) public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) { - // 1. 校验知识库是否存在 - validateKnowledgeParam(updateReqVO); - - // 2. 更新知识库主表基础信息 - KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); - knowledgeBaseMapper.updateById(updateObj); - - // 3. 处理附表(知识文档)数据 - handleKnowledgeDocuments(updateReqVO, updateObj); + // // 1. 校验知识库是否存在 + // validateKnowledgeParam(updateReqVO); + // + // // 2. 更新知识库主表基础信息 + // KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); + // knowledgeBaseMapper.updateById(updateObj); + // + // // 3. 处理附表(知识文档)数据 + // handleKnowledgeDocuments(updateReqVO, updateObj); } /** * 校验知识库参数 * - * @param updateReqVO 更新知识库参数 + * @param request 更新知识库参数 */ - private void validateKnowledgeParam (KnowledgeBaseSaveReqVO updateReqVO) { + private void validateKnowledgeParam (KnowledgeEmbeddingReqVO request) { // 1. 校验知识库是否存在 - validateKnowledgeBaseExists(updateReqVO.getId()); + validateKnowledgeBaseExists(request.getId()); - // 2. 校验知识库名称是否重复 - validateKnowledgeBaseNameExists(updateReqVO); - - // 3. 校验分块大小和分块重叠是否正确 - validateChunkParameters(updateReqVO.getChunkSize(), updateReqVO.getChunkOverlap()); + // 2. 校验分块大小和分块重叠是否正确 + validateChunkParameters(request.getChunkSize(), request.getChunkOverlap()); } /** @@ -109,36 +106,21 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { /** * 处理知识文档数据 * - * @param updateReqVO 更新知识库参数 - * @param updateObj 更新知识库对象 + * @param request 更新知识库参数 + * @param updateObj 更新知识库对象 */ - private void handleKnowledgeDocuments (KnowledgeBaseSaveReqVO updateReqVO, KnowledgeBaseDO updateObj) { - List documents = updateReqVO.getKnowledgeDocuments(); - if (CollectionUtils.isAnyEmpty(documents)) { - // 如果传入的文档列表为空,则删除所有关联文档 - deleteAllDocuments(updateReqVO.getId()); - return; - } + private void handleKnowledgeDocuments (KnowledgeEmbeddingReqVO request, KnowledgeBaseDO updateObj) { + // 1: 删除所有旧文档 + deleteAllDocuments(updateObj.getId()); - - // 获取需要保留的文档 ID - List retainedIds = documents.stream() - .map(KnowledgeDocumentsSaveReqVO::getId) - .collect(Collectors.toList()); - - // 删除不需要保留的文档 - deleteUnretainedDocuments(updateReqVO.getId(), retainedIds); - - // 更新或插入文档数据 - List newDocuments = updateOrInsertDocuments(documents, updateReqVO.getId(), updateObj.getKnowledgeLength()); + // 2: 重新上传文档 + List newDocuments = updateOrInsertDocuments(request.getDocuments(), request.getId()); Map knowledgeParameters = new HashMap<>(); - knowledgeParameters.put("chunkSize", updateReqVO.getChunkSize()); - knowledgeParameters.put("chunkOverlap", updateReqVO.getChunkOverlap()); + knowledgeParameters.put("chunkSize", request.getChunkSize()); + knowledgeParameters.put("chunkOverlap", request.getChunkOverlap()); - // 异步处理新增文档和删除的文档 - List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); - asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds, knowledgeParameters); + asyncKnowledgeBase.createKnowledgeBase(newDocuments, knowledgeParameters); } /** @@ -153,7 +135,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { // 异步处理删除的文档 List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(knowledgeBaseId); if (!CollectionUtils.isAnyEmpty(deleteIds)) { - asyncKnowledgeBase.createKnowledgeBase(new ArrayList<>(), deleteIds, new HashMap<>()); + asyncKnowledgeBase.deletedKnowledgeBase(deleteIds); } } @@ -177,10 +159,9 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { * * @param documents 需要更新的文档数据 * @param knowledgeBaseId 知识库 ID - * @param chunkSize * @return 更新或插入的文档数据 */ - private List updateOrInsertDocuments (List documents, Long knowledgeBaseId, Integer chunkSize) { + private List updateOrInsertDocuments (List documents, Long knowledgeBaseId) { List newDocuments = new ArrayList<>(); documents.forEach(doc -> { KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); @@ -315,17 +296,17 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { } Integer topK = 4; if (baseDO.getTopK() == null || baseDO.getTopK() <= 0) { - testReqVO.setK(topK); + testReqVO.setTopK(topK); } else { - topK = baseDO.getTopK(); - testReqVO.setK(topK); + topK = testReqVO.getTopK(); + testReqVO.setTopK(topK); } Double score = 0.2; if (baseDO.getScore() == null || baseDO.getTopK() <= 0.0 || baseDO.getScore() > 1) { testReqVO.setScore(score); } else { - score = baseDO.getScore(); + score = testReqVO.getScore(); testReqVO.setScore(score); } @@ -349,6 +330,39 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { return result; } + /** + * 知识库Embedding + * @param request + */ + @Override + public void embeddingKnowledge (KnowledgeEmbeddingReqVO request) { + // 1. 校验知识库是否存在 + validateKnowledgeParam(request); + + // 2. 更新知识库主表基础信息 + KnowledgeBaseDO updateObj = BeanUtils.toBean(request, KnowledgeBaseDO.class); + knowledgeBaseMapper.updateById(updateObj); + + // 3. 处理附表(知识文档)数据 + handleKnowledgeDocuments(request, updateObj); + } + + /** + * 更新知识库命中测试参数 + * @param updateReqVO + */ + @Override + public void updateHitParam (KnowledgeBaseSaveReqVO updateReqVO) { + // 校验存在 + validateKnowledgeBaseExists(updateReqVO.getId()); + // 更新 + KnowledgeBaseDO updateObj = new KnowledgeBaseDO(); + updateObj.setId(updateReqVO.getId()) + .setTopK(updateReqVO.getTopK()) + .setScore(updateReqVO.getScore()) + ; + knowledgeBaseMapper.updateById(updateObj); + } /** * 校验知识库是否存在