调试Ï
This commit is contained in:
parent
7a6c7558a1
commit
321047acc9
@ -2,9 +2,7 @@ package cn.iocoder.yudao.module.llm.service.async;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
|
||||
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
|
||||
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgebase.KnowledgeBaseDO;
|
||||
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
|
||||
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper;
|
||||
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
|
||||
import cn.iocoder.yudao.module.llm.service.http.RagHttpService;
|
||||
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
|
||||
@ -58,22 +56,19 @@ public class AsyncKnowledgeBase {
|
||||
});
|
||||
}
|
||||
}
|
||||
@Resource
|
||||
private KnowledgeBaseMapper knowledgeBaseMapper;
|
||||
|
||||
/**
|
||||
* 知识库向量嵌入
|
||||
*
|
||||
* @param knowledgeList 文件列表
|
||||
* @param id
|
||||
*/
|
||||
// @Async
|
||||
@Async
|
||||
public void knowledgeEmbed (List<KnowledgeDocumentsDO> knowledgeList, Long id) {
|
||||
if (!CollectionUtils.isAnyEmpty(knowledgeList)) {
|
||||
knowledgeList.forEach(knowledge -> {
|
||||
try {
|
||||
KnowledgeBaseDO aDo = knowledgeBaseMapper.selectById(id);
|
||||
aDo.setKnowledgeBaseIntro("5");
|
||||
knowledgeBaseMapper.updateById(aDo);
|
||||
|
||||
// TODO:本地调试时打开
|
||||
/*
|
||||
String tmpUrl = "http://xhllm.xinnuojinzhi.com/admin-api/infra/file/29/get/486b9a6fc855abf48847e9639f3c090855c6aafdc22a13b10e3244c37f03d3e0.txt";
|
||||
@ -86,8 +81,7 @@ public class AsyncKnowledgeBase {
|
||||
.setFileId(String.valueOf(knowledge.getId()))
|
||||
.setFileName(knowledge.getDocumentName())
|
||||
.setFileInputStream(new ByteArrayInputStream(Objects.requireNonNull(getFileByte(knowledge.getFileUrl()))));
|
||||
aDo.setKnowledgeBaseIntro("6");
|
||||
knowledgeBaseMapper.updateById(aDo);
|
||||
|
||||
ragHttpService.knowledgeEmbed(ragEmbedReqVo,id);
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
@ -16,7 +16,6 @@ import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper;
|
||||
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgedocuments.KnowledgeDocumentsMapper;
|
||||
import cn.iocoder.yudao.module.llm.service.application.ApplicationService;
|
||||
import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase;
|
||||
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import kong.unirest.Unirest;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@ -24,7 +23,6 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -65,56 +63,67 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
|
||||
|
||||
@Override
|
||||
public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) {
|
||||
// 校验存在
|
||||
// 1. 校验知识库是否存在
|
||||
validateKnowledgeBaseExists(updateReqVO.getId());
|
||||
// 2. 校验知识库名称是否重复
|
||||
validateKnowledgeBaseNameExists(updateReqVO);
|
||||
// 更新
|
||||
|
||||
// 3. 更新知识库主表
|
||||
KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class);
|
||||
knowledgeBaseMapper.updateById(updateObj);
|
||||
List<KnowledgeDocumentsDO> knowledgeDocumentsList = new ArrayList<>();
|
||||
// 附表增加数据
|
||||
|
||||
Unirest.config().reset();
|
||||
Unirest.config()
|
||||
.socketTimeout(86400000)
|
||||
.connectTimeout(100000)
|
||||
.concurrency(10, 5)
|
||||
.setDefaultHeader("Accept", "application/json")
|
||||
;
|
||||
|
||||
// 4. 处理附表（知识文档）数据
|
||||
if (!CollectionUtils.isAnyEmpty(updateReqVO.getKnowledgeDocuments())) {
|
||||
List<Long> ids = updateReqVO.getKnowledgeDocuments().stream().filter(
|
||||
knowledgeDocuments -> knowledgeDocuments.getId() != null
|
||||
).map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList());
|
||||
if (!CollectionUtils.isAnyEmpty(ids)) {
|
||||
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
|
||||
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())
|
||||
.notIn(KnowledgeDocumentsDO::getId, ids));
|
||||
} else {
|
||||
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
|
||||
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()));
|
||||
// 4.1 获取需要保留的文档 ID
|
||||
List<Long> retainedIds = updateReqVO.getKnowledgeDocuments().stream()
|
||||
.map(KnowledgeDocumentsSaveReqVO::getId)
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// 4.2 删除不需要保留的文档
|
||||
LambdaQueryWrapperX<KnowledgeDocumentsDO> deleteWrapper = new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
|
||||
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId());
|
||||
if (!CollectionUtils.isAnyEmpty(retainedIds)) {
|
||||
deleteWrapper.notIn(KnowledgeDocumentsDO::getId, retainedIds);
|
||||
}
|
||||
updateReqVO.getKnowledgeDocuments().forEach(
|
||||
knowledgeDocuments -> {
|
||||
KnowledgeDocumentsDO knowledgeDocumentsDO = BeanUtils.toBean(knowledgeDocuments, KnowledgeDocumentsDO.class);
|
||||
knowledgeDocumentsDO.setKnowledgeBaseId(updateReqVO.getId());
|
||||
if (knowledgeDocuments.getId() == null) {
|
||||
knowledgeDocumentsList.add(knowledgeDocumentsDO);
|
||||
}
|
||||
knowledgeDocumentsMapper.insertOrUpdate(knowledgeDocumentsDO);
|
||||
}
|
||||
);
|
||||
knowledgeDocumentsMapper.delete(deleteWrapper);
|
||||
|
||||
// 4.3 更新或插入文档数据
|
||||
List<KnowledgeDocumentsDO> newDocuments = new ArrayList<>();
|
||||
updateReqVO.getKnowledgeDocuments().forEach(doc -> {
|
||||
KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class);
|
||||
docDO.setKnowledgeBaseId(updateReqVO.getId());
|
||||
if (doc.getId() == null) {
|
||||
newDocuments.add(docDO); // 收集新增文档
|
||||
}
|
||||
knowledgeDocumentsMapper.insertOrUpdate(docDO); // 更新或插入文档
|
||||
});
|
||||
|
||||
// 4.4 异步处理新增文档和删除的文档
|
||||
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
|
||||
asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds);
|
||||
// 4.5 异步处理知识库外挂
|
||||
asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
|
||||
// List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
|
||||
// asyncKnowledgeBase.createKnowledgeBase(knowledgeDocumentsList, deleteIds);
|
||||
// updateObj.setKnowledgeBaseIntro("4");
|
||||
// knowledgeBaseMapper.updateById(updateObj);
|
||||
// 4.5 异步处理知识库外挂
|
||||
// asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
|
||||
asyncKnowledgeBase.knowledgeEmbed(newDocuments,updateReqVO.getId());
|
||||
} else {
|
||||
// 5. 如果传入的文档列表为空，则删除所有关联文档
|
||||
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
|
||||
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()));
|
||||
|
||||
// 5.1 异步处理删除的文档
|
||||
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
|
||||
if (!CollectionUtils.isAnyEmpty(deleteIds)) {
|
||||
asyncKnowledgeBase.createKnowledgeBase(null, deleteIds);
|
||||
// 4.5 异步处理知识库外挂
|
||||
asyncKnowledgeBase.knowledgeEmbed(knowledgeDocumentsList, updateReqVO.getId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user