From 321047acc90be7e101d6bf6d1ef344da2fdd7b39 Mon Sep 17 00:00:00 2001 From: Liuyang <2746366019@qq.com> Date: Mon, 10 Feb 2025 12:32:52 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E8=AF=95=C3=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../llm/service/async/AsyncKnowledgeBase.java | 14 +--- .../KnowledgeBaseServiceImpl.java | 81 ++++++++++--------- 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 3408e9964..f806b747d 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -2,9 +2,7 @@ package cn.iocoder.yudao.module.llm.service.async; import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; -import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgebase.KnowledgeBaseDO; import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; -import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.RagHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; @@ -58,22 +56,19 @@ public class AsyncKnowledgeBase { }); } } - @Resource - private KnowledgeBaseMapper knowledgeBaseMapper; + /** * 知识库向量嵌入 * * @param knowledgeList 文件列表 * @param id */ -// @Async + @Async public void knowledgeEmbed (List knowledgeList, Long id) { if (!CollectionUtils.isAnyEmpty(knowledgeList)) { knowledgeList.forEach(knowledge -> { try { - KnowledgeBaseDO aDo = knowledgeBaseMapper.selectById(id); - aDo.setKnowledgeBaseIntro("5"); - knowledgeBaseMapper.updateById(aDo); + // TODO:本地调试时打开 /* String tmpUrl = "http://xhllm.xinnuojinzhi.com/admin-api/infra/file/29/get/486b9a6fc855abf48847e9639f3c090855c6aafdc22a13b10e3244c37f03d3e0.txt"; @@ -86,8 +81,7 @@ public class AsyncKnowledgeBase { .setFileId(String.valueOf(knowledge.getId())) .setFileName(knowledge.getDocumentName()) .setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl())))); - aDo.setKnowledgeBaseIntro("6"); - knowledgeBaseMapper.updateById(aDo); + ragHttpService.knowledgeEmbed(ragEmbedReqVo,id); } catch (Exception e) { diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 64b210c58..afbc4dd68 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -16,7 +16,6 @@ import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper; import cn.iocoder.yudao.module.llm.dal.mysql.knowledgedocuments.KnowledgeDocumentsMapper; import cn.iocoder.yudao.module.llm.service.application.ApplicationService; import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase; -import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import kong.unirest.Unirest; import org.springframework.beans.factory.annotation.Autowired; @@ -24,7 +23,6 @@ import org.springframework.stereotype.Service; import org.springframework.validation.annotation.Validated; import javax.annotation.Resource; -import java.io.ByteArrayInputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -65,56 +63,67 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { @Override public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) { - // 校验存在 + // 1. 校验知识库是否存在 validateKnowledgeBaseExists(updateReqVO.getId()); + // 2. 校验知识库名称是否重复 validateKnowledgeBaseNameExists(updateReqVO); - // 更新 + + // 3. 更新知识库主表 KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); knowledgeBaseMapper.updateById(updateObj); - List knowledgeDocumentsList = new ArrayList<>(); - // 附表增加数据 + + Unirest.config().reset(); + Unirest.config() + .socketTimeout(86400000) + .connectTimeout(100000) + .concurrency(10, 5) + .setDefaultHeader("Accept", "application/json") + ; + + // 4. 处理附表（知识文档）数据 if (!CollectionUtils.isAnyEmpty(updateReqVO.getKnowledgeDocuments())) { - List ids = updateReqVO.getKnowledgeDocuments().stream().filter( - knowledgeDocuments -> knowledgeDocuments.getId() != null - ).map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList()); - if (!CollectionUtils.isAnyEmpty(ids)) { - knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX() - .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()) - .notIn(KnowledgeDocumentsDO::getId, ids)); - } else { - knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX() - .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())); + // 4.1 获取需要保留的文档 ID + List retainedIds = updateReqVO.getKnowledgeDocuments().stream() + .map(KnowledgeDocumentsSaveReqVO::getId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + // 4.2 删除不需要保留的文档 + LambdaQueryWrapperX deleteWrapper = new LambdaQueryWrapperX() + .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()); + if (!CollectionUtils.isAnyEmpty(retainedIds)) { + deleteWrapper.notIn(KnowledgeDocumentsDO::getId, retainedIds); } - updateReqVO.getKnowledgeDocuments().forEach( - knowledgeDocuments -> { - KnowledgeDocumentsDO knowledgeDocumentsDO = BeanUtils.toBean(knowledgeDocuments, KnowledgeDocumentsDO.class); - knowledgeDocumentsDO.setKnowledgeBaseId(updateReqVO.getId()); - if (knowledgeDocuments.getId() == null) { - knowledgeDocumentsList.add(knowledgeDocumentsDO); - } - knowledgeDocumentsMapper.insertOrUpdate(knowledgeDocumentsDO); - } - ); + knowledgeDocumentsMapper.delete(deleteWrapper); + + // 4.3 更新或插入文档数据 + List newDocuments = new ArrayList<>(); + updateReqVO.getKnowledgeDocuments().forEach(doc -> { + KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class); + docDO.setKnowledgeBaseId(updateReqVO.getId()); + if (doc.getId() == null) { + newDocuments.add(docDO); // 收集新增文档 + } + knowledgeDocumentsMapper.insertOrUpdate(docDO); // 更新或插入文档 + }); + + // 4.4 异步处理新增文档和删除的文档 + List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); + asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds); // 4.5 异步处理知识库外挂 - asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId()); -// List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); -// asyncKnowledgeBase.createKnowledgeBase(knowledgeDocumentsList, deleteIds); -// updateObj.setKnowledgeBaseIntro("4"); -// knowledgeBaseMapper.updateById(updateObj); - // 4.5 异步处理知识库外挂 -// asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId()); + asyncKnowledgeBase.knowledgeEmbed(newDocuments,updateReqVO.getId()); } else { + // 5. 如果传入的文档列表为空，则删除所有关联文档 knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX() .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())); + + // 5.1 异步处理删除的文档 List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); if (!CollectionUtils.isAnyEmpty(deleteIds)) { asyncKnowledgeBase.createKnowledgeBase(null, deleteIds); - // 4.5 异步处理知识库外挂 - asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId()); } } - } @Override