From b056abf5138f54d60cf10ccd6d6a22d13afe2689 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Mon, 10 Mar 2025 16:39:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(module-llm):=E5=A2=9E=E5=8A=A0=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E6=96=87=E6=A1=A3=E5=88=86=E5=9D=97=E5=A4=A7?= =?UTF-8?q?=E5=B0=8F=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 KnowledgeBaseSaveReqVO 中添加 chunkSize 字段，用于设置分块大小 - 在 KnowledgeDocumentsDO 中添加 chunkSize 字段，用于存储分块大小- 修改 KnowledgeBaseServiceImpl，将 chunkSize 设置为知识文档的分块大小 - 在 KnowledgeRagEmbedReqVO 中添加 chunkSize 字段，用于传递分块大小 - 修改 RagHttpService，将 chunkSize 添加到请求体中 --- .../admin/knowledgebase/vo/KnowledgeBaseSaveReqVO.java | 7 ++++++- .../knowledgedocuments/KnowledgeDocumentsDO.java | 5 ++++- .../yudao/module/llm/service/http/RagHttpService.java | 5 ++++- .../module/llm/service/http/vo/KnowledgeRagEmbedReqVO.java | 5 +++++ .../service/knowledgebase/KnowledgeBaseServiceImpl.java | 1 + 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBaseSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBaseSaveReqVO.java index 40177bc6f..0c5d43d69 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBaseSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBaseSaveReqVO.java @@ -32,9 +32,14 @@ public class KnowledgeBaseSaveReqVO { @Schema(description = "知识长度") private Integer knowledgeLength; + /** + * 分块大小 + */ + private Integer chunkSize; + @Schema(description = "文件引用") private String knowledgeFile; @Schema(description = "文件引用上传") private List knowledgeDocuments; -} \ No newline at end of file +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/knowledgedocuments/KnowledgeDocumentsDO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/knowledgedocuments/KnowledgeDocumentsDO.java index 878893a51..854a28c7b 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/knowledgedocuments/KnowledgeDocumentsDO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/dataobject/knowledgedocuments/KnowledgeDocumentsDO.java @@ -48,5 +48,8 @@ public class KnowledgeDocumentsDO extends BaseDO { * 文件Id */ private Long fileId; - + /** + * 分块大小 + */ + private Integer chunkSize; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java index ac683ba3e..4380fdb3f 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java @@ -51,6 +51,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; @@ -388,9 +389,10 @@ public class RagHttpService { String fileId = reqVO.getFileId(); String fileName = reqVO.getFileName(); String fileUrl = reqVO.getFileUrl(); + Integer chunkSize = Optional.ofNullable(reqVO.getChunkSize()).orElse(1500); String mediaType = getMediaType(fileName); - log.info("文件ID: {}, 文件名: {}, 文件URL: {}, 文件类型: {}", fileId, fileName, fileUrl, mediaType); + log.info("文件ID: {}, 文件名: {}, 文件URL: {}, 文件类型: {}, 分块大小:{}", fileId, fileName, fileUrl, mediaType,chunkSize); // 获取知识库文档 log.info("开始获取知识库文档，知识库ID: {}, 文件ID: {}", id, fileId); @@ -447,6 +449,7 @@ public class RagHttpService { RequestBody requestBody = new MultipartBody.Builder() .setType(MultipartBody.FORM) .addFormDataPart("file_id", fileId) + .addFormDataPart("chunk_size", String.valueOf(chunkSize)) .addFormDataPart("file", fileName, RequestBody.create(tempFilePath.toFile(), MediaType.parse(mediaType)) ) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/KnowledgeRagEmbedReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/KnowledgeRagEmbedReqVO.java index 708f0f406..646c3a2bb 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/KnowledgeRagEmbedReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/KnowledgeRagEmbedReqVO.java @@ -41,4 +41,9 @@ public class KnowledgeRagEmbedReqVO { * 文件字节数组 */ private byte[] fileBytes; + + /** + * 分块大小 + */ + private Integer chunkSize; } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 397fcfc76..3229b0bfb 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -106,6 +106,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { updateReqVO.getKnowledgeDocuments().forEach(doc -> { KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); docDO.setKnowledgeBaseId(updateReqVO.getId()); + docDO.setChunkSize(updateObj.getKnowledgeLength()); if (doc.getId() == null) { newDocuments.add(docDO); // 收集新增文档 }