DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source 
# Conflicts:
#	yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java
This commit is contained in:
Liuyang 2025-02-10 12:28:13 +08:00
commit 7a6c7558a1
2 changed files with 46 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java
View File

@ -2,7 +2,9 @@ package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgebase.KnowledgeBaseDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.RagHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
@ -56,19 +58,22 @@ public class AsyncKnowledgeBase {
});
}
}
@Resource
private KnowledgeBaseMapper knowledgeBaseMapper;
/**
* 知识库向量嵌入
*
* @param knowledgeList 文件列表
* @param id
*/
@Async
// @Async
public void knowledgeEmbed (List<KnowledgeDocumentsDO> knowledgeList, Long id) {
if (!CollectionUtils.isAnyEmpty(knowledgeList)) {
knowledgeList.forEach(knowledge -> {
try {
KnowledgeBaseDO aDo = knowledgeBaseMapper.selectById(id);
aDo.setKnowledgeBaseIntro("5");
knowledgeBaseMapper.updateById(aDo);
// TODO:本地调试时打开
/*
String tmpUrl = "http://xhllm.xinnuojinzhi.com/admin-api/infra/file/29/get/486b9a6fc855abf48847e9639f3c090855c6aafdc22a13b10e3244c37f03d3e0.txt";
@ -81,7 +86,8 @@ public class AsyncKnowledgeBase {
.setFileId(String.valueOf(knowledge.getId()))
.setFileName(knowledge.getDocumentName())
.setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl()))));
aDo.setKnowledgeBaseIntro("6");
knowledgeBaseMapper.updateById(aDo);
ragHttpService.knowledgeEmbed(ragEmbedReqVo,id);
} catch (Exception e) {

81
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java
View File

@ -16,6 +16,7 @@ import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgedocuments.KnowledgeDocumentsMapper;
import cn.iocoder.yudao.module.llm.service.application.ApplicationService;
import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase;
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import kong.unirest.Unirest;
import org.springframework.beans.factory.annotation.Autowired;
@ -23,6 +24,7 @@ import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
import javax.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -63,67 +65,56 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
@Override
public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) {
// 1. 校验知识库是否存在
// 校验存在
validateKnowledgeBaseExists(updateReqVO.getId());
// 2. 校验知识库名称是否重复
validateKnowledgeBaseNameExists(updateReqVO);
// 3. 更新知识库主表
// 更新
KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class);
knowledgeBaseMapper.updateById(updateObj);
Unirest.config().reset();
Unirest.config()
.socketTimeout(86400000)
.connectTimeout(100000)
.concurrency(10, 5)
.setDefaultHeader("Accept", "application/json")
;
// 4. 处理附表（知识文档）数据
List<KnowledgeDocumentsDO> knowledgeDocumentsList = new ArrayList<>();
// 附表增加数据
if (!CollectionUtils.isAnyEmpty(updateReqVO.getKnowledgeDocuments())) {
// 4.1 获取需要保留的文档 ID
List<Long> retainedIds = updateReqVO.getKnowledgeDocuments().stream()
.map(KnowledgeDocumentsSaveReqVO::getId)
.filter(Objects::nonNull)
.collect(Collectors.toList());
// 4.2 删除不需要保留的文档
LambdaQueryWrapperX<KnowledgeDocumentsDO> deleteWrapper = new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId());
if (!CollectionUtils.isAnyEmpty(retainedIds)) {
deleteWrapper.notIn(KnowledgeDocumentsDO::getId, retainedIds);
List<Long> ids = updateReqVO.getKnowledgeDocuments().stream().filter(
knowledgeDocuments -> knowledgeDocuments.getId() != null
).map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList());
if (!CollectionUtils.isAnyEmpty(ids)) {
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())
.notIn(KnowledgeDocumentsDO::getId, ids));
} else {
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()));
}
knowledgeDocumentsMapper.delete(deleteWrapper);
// 4.3 更新或插入文档数据
List<KnowledgeDocumentsDO> newDocuments = new ArrayList<>();
updateReqVO.getKnowledgeDocuments().forEach(doc -> {
KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class);
docDO.setKnowledgeBaseId(updateReqVO.getId());
if (doc.getId() == null) {
newDocuments.add(docDO); // 收集新增文档
}
knowledgeDocumentsMapper.insertOrUpdate(docDO); // 更新或插入文档
});
// 4.4 异步处理新增文档和删除的文档
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds);
updateReqVO.getKnowledgeDocuments().forEach(
knowledgeDocuments -> {
KnowledgeDocumentsDO knowledgeDocumentsDO = BeanUtils.toBean(knowledgeDocuments, KnowledgeDocumentsDO.class);
knowledgeDocumentsDO.setKnowledgeBaseId(updateReqVO.getId());
if (knowledgeDocuments.getId() == null) {
knowledgeDocumentsList.add(knowledgeDocumentsDO);
}
knowledgeDocumentsMapper.insertOrUpdate(knowledgeDocumentsDO);
}
);
// 4.5 异步处理知识库外挂
asyncKnowledgeBase.knowledgeEmbed(newDocuments,updateReqVO.getId());
asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
// List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
// asyncKnowledgeBase.createKnowledgeBase(knowledgeDocumentsList, deleteIds);
// updateObj.setKnowledgeBaseIntro("4");
// knowledgeBaseMapper.updateById(updateObj);
// 4.5 异步处理知识库外挂
// asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
} else {
// 5. 如果传入的文档列表为空，则删除所有关联文档
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()));
// 5.1 异步处理删除的文档
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
if (!CollectionUtils.isAnyEmpty(deleteIds)) {
asyncKnowledgeBase.createKnowledgeBase(null, deleteIds);
// 4.5 异步处理知识库外挂
asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
}
}
}
@Override