DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

refactor(llm): 重构知识库异步处理逻辑

Browse Source 
-移除了 Async 注解，改为在服务层使用 @Transactional 注解- 优化了知识库嵌入逻辑，改为单个文件处理- 删除了未使用的代码片段- 调整了日志输出位置
This commit is contained in:
Liuyang 2025-02-12 14:50:11 +08:00
parent acfa7524e0
commit 85f3825ae7
2 changed files with 22 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java
View File

@ -40,7 +40,6 @@ public class AsyncKnowledgeBase {
// 向向量知识库创建文件
@Async
public void createKnowledgeBase (List<KnowledgeDocumentsDO> knowledgeList, List<Long> ids) {
if (!CollectionUtils.isAnyEmpty(ids)) {
String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids);
@ -69,11 +68,7 @@ public class AsyncKnowledgeBase {
if ("txt".equals(extension)) {
ragHttpService.embedUploadFile(regUploadReqVO);
} else {
KnowledgeRagEmbedReqVO knowledgeRagEmbedReqVO = new KnowledgeRagEmbedReqVO()
.setFileId(String.valueOf(knowledge.getId()))
.setFileName(knowledge.getDocumentName())
.setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl()))));
ragHttpService.knowledgeEmbed(knowledgeRagEmbedReqVO, knowledge.getKnowledgeBaseId());
knowledgeEmbed(knowledge, knowledge.getKnowledgeBaseId());
}
}
@ -102,34 +97,28 @@ public class AsyncKnowledgeBase {
/**
* 知识库向量嵌入
*
* @param knowledgeList 文件列表
* @param id
* @param knowledge 文件
* @param id 知识库id
*/
@Async
public void knowledgeEmbed (List<KnowledgeDocumentsDO> knowledgeList, Long id) {
if (!CollectionUtils.isAnyEmpty(knowledgeList)) {
knowledgeList.forEach(knowledge -> {
try {
public void knowledgeEmbed (KnowledgeDocumentsDO knowledge, Long id) {
// TODO:本地调试时打开
/*
String tmpUrl = "http://xhllm.xinnuojinzhi.com/admin-api/infra/file/29/get/486b9a6fc855abf48847e9639f3c090855c6aafdc22a13b10e3244c37f03d3e0.txt";
log.info("knowledge url {}", tmpUrl);
knowledge.setFileUrl(tmpUrl);
*/
try {
// 创建知识向量
KnowledgeRagEmbedReqVO ragEmbedReqVo = new KnowledgeRagEmbedReqVO()
.setFileId(String.valueOf(knowledge.getId()))
.setFileName(knowledge.getDocumentName())
.setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl()))));
// TODO:本地调试时打开
// String tmpUrl = "http://xhllm.xinnuojinzhi.com/admin-api/infra/file/29/get/ca3d06d24f80c127ec0300408a035176f5e0bf90ce319fda17018303226e2298.doc";
// log.info("knowledge url {}", tmpUrl);
// knowledge.setFileUrl(tmpUrl);
ragHttpService.knowledgeEmbed(ragEmbedReqVo, id);
// 创建知识向量
KnowledgeRagEmbedReqVO ragEmbedReqVo = new KnowledgeRagEmbedReqVO()
.setFileId(String.valueOf(knowledge.getId()))
.setFileName(knowledge.getDocumentName())
.setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl()))));
} catch (Exception e) {
log.error("the creation of the knowledge base error {}", e.getMessage(), e);
}
});
ragHttpService.knowledgeEmbed(ragEmbedReqVo, id);
} catch (Exception e) {
log.error("the creation of the knowledge base error {}", e.getMessage(), e);
}
}
@ -140,8 +129,6 @@ public class AsyncKnowledgeBase {
* @return 文件字节数组
*/
public static byte[] getFileByte (String fileUrl) {
log.info("knowledge url: {}", fileUrl);
try (InputStream inputStream = new URL(fileUrl).openStream();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {

6
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java
View File

@ -20,9 +20,11 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import kong.unirest.Unirest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import javax.annotation.Resource;
import javax.annotation.Tainted;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -62,9 +64,11 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
}
@Override
@Transactional
public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) {
// 1. 校验知识库是否存在
validateKnowledgeBaseExists(updateReqVO.getId());
// 2. 校验知识库名称是否重复
validateKnowledgeBaseNameExists(updateReqVO);
@ -109,8 +113,6 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
// 4.4 异步处理新增文档和删除的文档
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds);
// 4.5 异步处理知识库外挂
// asyncKnowledgeBase.knowledgeEmbed(newDocuments,updateReqVO.getId());
} else {
// 5. 如果传入的文档列表为空，则删除所有关联文档
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()