DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
sunxiqing 2025-03-14 16:24:26 +08:00
commit 0e6e306b2e
10 changed files with 189 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java
View File

@ -55,6 +55,14 @@ public class KnowledgeBaseController {
return success(result);
}
@PutMapping("/updateHitParam")
@Operation(summary = "更新知识库命中测试参数")
public CommonResult<Boolean> updateHitParam(@Valid @RequestBody KnowledgeBaseSaveReqVO updateReqVO) {
knowledgeBaseService.updateHitParam(updateReqVO);
return success(true);
}
@PutMapping("/update")
@Operation(summary = "更新知识库")
// @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')")
@ -63,6 +71,13 @@ public class KnowledgeBaseController {
return success(true);
}
@PutMapping("/embeddingKnowledge")
@Operation(summary = "更新 Embedding", description = "对文件内容进行 Embedding 处理并更新到知识库")
public CommonResult<Boolean> embeddingKnowledge(@Valid @RequestBody KnowledgeEmbeddingReqVO request) {
knowledgeBaseService.embeddingKnowledge(request);
return success(true);
}
@PutMapping("/updateKnowledgeBaseInfo")
@Operation(summary = "更新知识库")
// @PreAuthorize("@ss.hasPermission('llm:knowledge-base:update')")

30
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeEmbeddingReqVO.java Normal file
View File

@ -0,0 +1,30 @@
package cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsSaveReqVO;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
/**
* @Description 知识库 Embedding 请求参数
* @Date 2025/3/14 13:05
*/
@Schema(description = "知识库 Embedding 请求参数")
@Data
public class KnowledgeEmbeddingReqVO {
@Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "25984")
private Long id;
@Schema(description = "知识内容长度", example = "1000")
private Integer contentLength;
@Schema(description = "分块大小（单位：字符）", example = "512")
private Integer chunkSize;
@Schema(description = "分块重叠大小（单位：字符）", example = "128")
private Integer chunkOverlap;
@Schema(description = "关联的知识文档列表")
private List<KnowledgeDocumentsSaveReqVO> knowledgeDocuments;
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeHitRateTestReqVO.java
View File

@ -25,7 +25,7 @@ public class KnowledgeHitRateTestReqVO {
* 返回结果的条数（k值）
*/
// @NotNull(message = "k值不能为空")
private Integer k;
private Integer topK;
/**
* Score阈值

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java
View File

@ -13,7 +13,6 @@ public class KnowledgeDocumentsSaveReqVO {
private Long id;
@Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "18229")
// @NotNull(message = "知识库ID不能为空")
private Long knowledgeBaseId;
@Schema(description = "文档名称", example = "芋艿")
@ -28,4 +27,4 @@ public class KnowledgeDocumentsSaveReqVO {
@Schema(description = "文件id", example = "1")
private Long fileId;
}
}

5
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java
View File

@ -28,7 +28,6 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX<KnowledgeDocuments
.orderByDesc(KnowledgeDocumentsDO::getId));
}
@Select("SELECT id FROM llm_knowledge_documents WHERE knowledge_base_id = #{id} and deleted = 1")
@Select("SELECT file_id FROM llm_knowledge_documents WHERE knowledge_base_id = #{id} and deleted = 1")
List<Long> selectDeleteIds(Long id);
}
}

47
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java
View File

@ -10,9 +10,9 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.RagHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.RegUploadReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -23,7 +23,6 @@ import java.io.IOException;
import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -44,24 +43,12 @@ public class AsyncKnowledgeBase {
private KnowledgeDocumentsMapper knowledgeDocumentsMapper;
// 向向量知识库创建文件
// @Async
public void createKnowledgeBase (List<KnowledgeDocumentsDO> knowledgeList, List<Long> ids, Map<String, Integer> knowledgeParameters) {
log.info("开始执行 createKnowledgeBase 方法。knowledgeList 大小: {}, ids 大小: {}", knowledgeList.size(), ids.size());
// 如果提供了 ids，则删除现有的知识库文档
if (!CollectionUtils.isAnyEmpty(ids)) {
log.info("正在删除现有的知识库文档，ids: {}", ids);
try {
String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids);
log.info("删除知识库信息: {}", mes);
} catch (Exception e) {
log.error("删除知识库信息失败: {}", e.getMessage());
throw exception(new ErrorCode(10047, "删除知识库信息失败！"));
}
} else {
log.info("未提供 ids，跳过删除操作。");
}
/**
* 向向量知识库创建文件
* @param knowledgeList
* @param knowledgeParameters
*/
public void createKnowledgeBase (List<KnowledgeDocumentsDO> knowledgeList, Map<String, Integer> knowledgeParameters) {
// 处理 knowledgeList 中的每个知识文档
if (!CollectionUtils.isAnyEmpty(knowledgeList)) {
@ -100,6 +87,24 @@ public class AsyncKnowledgeBase {
log.info("createKnowledgeBase 方法执行完成。");
}
public void deletedKnowledgeBase ( List<Long> ids) {
log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids));
// 如果提供了 ids，则删除现有的知识库文档
if (!CollectionUtils.isAnyEmpty(ids)) {
log.info("正在删除现有的知识库文档，ids: {}", ids);
try {
String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids);
log.info("删除知识库信息: {}", mes);
} catch (Exception e) {
log.error("删除知识库信息失败: {}", e.getMessage());
throw exception(new ErrorCode(10047, "删除知识库信息失败！"));
}
} else {
log.info("未提供 ids，跳过删除操作。");
}
}
/**
* 修改知识库文档状态
*
@ -143,7 +148,7 @@ public class AsyncKnowledgeBase {
QueryMultipleReqVO vo = new QueryMultipleReqVO();
vo.setQuery(testReqVO.getQuery());
vo.setFileIds(fileIdStr);
vo.setK(testReqVO.getK());
vo.setK(testReqVO.getTopK());
vo.setScore(testReqVO.getScore());
List<KnowledgeHitRateTestResultVO> resultList = new ArrayList<>();

54
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java
View File

@ -106,33 +106,33 @@ public class BaseModelTaskService {
String status = latestRecord.getStatus();
log.info("最新模型部署记录状态: {}", status);
// 如果模型状态为 "stop"，则重新部署
if ("stop".equals(status)) {
log.info("模型状态为 'stop'，正在重新部署模型...");
// 构建模型部署请求
AigcModelDeploySaveReq aigcModelDeploySaveReq = new AigcModelDeploySaveReq(
baseModelDO.getAigcModelName(), "gpu");
log.info("模型部署请求参数: {}", JSON.toJSONString(aigcModelDeploySaveReq));
// 发起模型部署请求
ModelDeployRespVO modelDeployRespVO = trainHttpService.modelDeploy(
new HashMap<>(), serverName.getHost(), aigcModelDeploySaveReq);
log.info("模型部署请求完成。响应内容: {}", JSON.toJSONString(modelDeployRespVO));
// 更新基础模型信息
if (!"error".equals(modelDeployRespVO.getMessage())) {
log.info("模型部署成功。正在更新基础模型信息...");
BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO();
baseModelSaveReqVO.setId(baseModelDO.getId());
baseModelSaveReqVO.setModelId(modelDeployRespVO.getId());
baseModelSaveReqVO.setChatUrl(modelDeployRespVO.getPort() + DEFAULT_MODEL_URL_SUFFIX);
baseModelService.updateBaseModel(baseModelSaveReqVO);
log.info("基础模型信息更新完成。模型ID: {}", baseModelDO.getId());
} else {
log.error("模型部署失败。模型ID: {}", baseModelDO.getId());
}
}
// // 如果模型状态为 "stop"，则重新部署
// if ("stop".equals(status)) {
// log.info("模型状态为 'stop'，正在重新部署模型...");
//
// // 构建模型部署请求
// AigcModelDeploySaveReq aigcModelDeploySaveReq = new AigcModelDeploySaveReq(
// baseModelDO.getAigcModelName(), "gpu");
// log.info("模型部署请求参数: {}", JSON.toJSONString(aigcModelDeploySaveReq));
//
// // 发起模型部署请求
// ModelDeployRespVO modelDeployRespVO = trainHttpService.modelDeploy(
// new HashMap<>(), serverName.getHost(), aigcModelDeploySaveReq);
// log.info("模型部署请求完成。响应内容: {}", JSON.toJSONString(modelDeployRespVO));
//
// // 更新基础模型信息
// if (!"error".equals(modelDeployRespVO.getMessage())) {
// log.info("模型部署成功。正在更新基础模型信息...");
// BaseModelSaveReqVO baseModelSaveReqVO = new BaseModelSaveReqVO();
// baseModelSaveReqVO.setId(baseModelDO.getId());
// baseModelSaveReqVO.setModelId(modelDeployRespVO.getId());
// baseModelSaveReqVO.setChatUrl(modelDeployRespVO.getPort() + DEFAULT_MODEL_URL_SUFFIX);
// baseModelService.updateBaseModel(baseModelSaveReqVO);
// log.info("基础模型信息更新完成。模型ID: {}", baseModelDO.getId());
// } else {
// log.error("模型部署失败。模型ID: {}", baseModelDO.getId());
// }
// }
} else {
log.warn("未找到模型部署记录。模型ID: {}", modelId);
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java
View File

@ -723,7 +723,7 @@ public class RagHttpService {
private KnowledgeDocumentsDO getKnowledgeDocuments (Long id, String fileId) {
return knowledgeDocumentsMapper.selectOne(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, id)
.eq(KnowledgeDocumentsDO::getId, fileId));
.eq(KnowledgeDocumentsDO::getFileId, fileId));
}
/**

12
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseService.java
View File

@ -75,4 +75,16 @@ public interface KnowledgeBaseService {
* @return 返回结果
*/
List<KnowledgeHitRateTestResultVO> executeHitRateTest (@Valid KnowledgeHitRateTestReqVO testReqVO);
/**
* 知识库Embedding
* @param request
*/
void embeddingKnowledge (@Valid KnowledgeEmbeddingReqVO request);
/**
* 更新知识库命中测试参数
* @param updateReqVO
*/
void updateHitParam (@Valid KnowledgeBaseSaveReqVO updateReqVO);
}

127
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java
View File

@ -60,31 +60,28 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
@Override
@Transactional(rollbackFor = Exception.class)
public void updateKnowledgeBase (KnowledgeBaseSaveReqVO updateReqVO) {
// 1. 校验知识库是否存在
validateKnowledgeParam(updateReqVO);
// 2. 更新知识库主表基础信息
KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class);
knowledgeBaseMapper.updateById(updateObj);
// 3. 处理附表（知识文档）数据
handleKnowledgeDocuments(updateReqVO, updateObj);
// // 1. 校验知识库是否存在
// validateKnowledgeParam(updateReqVO);
//
// // 2. 更新知识库主表基础信息
// KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class);
// knowledgeBaseMapper.updateById(updateObj);
//
// // 3. 处理附表（知识文档）数据
// handleKnowledgeDocuments(updateReqVO, updateObj);
}
/**
* 校验知识库参数
*
* @param updateReqVO 更新知识库参数
* @param request 更新知识库参数
*/
private void validateKnowledgeParam (KnowledgeBaseSaveReqVO updateReqVO) {
private void validateKnowledgeParam (KnowledgeEmbeddingReqVO request) {
// 1. 校验知识库是否存在
validateKnowledgeBaseExists(updateReqVO.getId());
validateKnowledgeBaseExists(request.getId());
// 2. 校验知识库名称是否重复
validateKnowledgeBaseNameExists(updateReqVO);
// 3. 校验分块大小和分块重叠是否正确
validateChunkParameters(updateReqVO.getChunkSize(), updateReqVO.getChunkOverlap());
// 2. 校验分块大小和分块重叠是否正确
validateChunkParameters(request.getChunkSize(), request.getChunkOverlap());
}
/**
@ -109,36 +106,21 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
/**
* 处理知识文档数据
*
* @param updateReqVO 更新知识库参数
* @param updateObj 更新知识库对象
* @param request 更新知识库参数
* @param updateObj 更新知识库对象
*/
private void handleKnowledgeDocuments (KnowledgeBaseSaveReqVO updateReqVO, KnowledgeBaseDO updateObj) {
List<KnowledgeDocumentsSaveReqVO> documents = updateReqVO.getKnowledgeDocuments();
if (CollectionUtils.isAnyEmpty(documents)) {
// 如果传入的文档列表为空，则删除所有关联文档
deleteAllDocuments(updateReqVO.getId());
return;
}
private void handleKnowledgeDocuments (KnowledgeEmbeddingReqVO request, KnowledgeBaseDO updateObj) {
// 1: 删除所有旧文档
deleteAllDocuments(updateObj.getId());
// 获取需要保留的文档 ID
List<Long> retainedIds = documents.stream()
.map(KnowledgeDocumentsSaveReqVO::getId)
.filter(Objects::nonNull)
.collect(Collectors.toList());
// 删除不需要保留的文档
deleteUnretainedDocuments(updateReqVO.getId(), retainedIds);
// 更新或插入文档数据
List<KnowledgeDocumentsDO> newDocuments = updateOrInsertDocuments(documents, updateReqVO.getId(), updateObj.getKnowledgeLength());
// 2: 重新上传文档
List<KnowledgeDocumentsDO> newDocuments = updateOrInsertDocuments(request.getKnowledgeDocuments(), request.getId());
Map<String, Integer> knowledgeParameters = new HashMap<>();
knowledgeParameters.put("chunkSize", updateReqVO.getChunkSize());
knowledgeParameters.put("chunkOverlap", updateReqVO.getChunkOverlap());
knowledgeParameters.put("chunkSize", request.getChunkSize());
knowledgeParameters.put("chunkOverlap", request.getChunkOverlap());
// 异步处理新增文档和删除的文档
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
asyncKnowledgeBase.createKnowledgeBase(newDocuments, deleteIds, knowledgeParameters);
asyncKnowledgeBase.createKnowledgeBase(newDocuments, knowledgeParameters);
}
/**
@ -153,7 +135,7 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
// 异步处理删除的文档
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(knowledgeBaseId);
if (!CollectionUtils.isAnyEmpty(deleteIds)) {
asyncKnowledgeBase.createKnowledgeBase(new ArrayList<>(), deleteIds, new HashMap<>());
asyncKnowledgeBase.deletedKnowledgeBase(deleteIds);
}
}
@ -177,18 +159,18 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
*
* @param documents 需要更新的文档数据
* @param knowledgeBaseId 知识库 ID
* @param chunkSize
* @return 更新或插入的文档数据
*/
private List<KnowledgeDocumentsDO> updateOrInsertDocuments (List<KnowledgeDocumentsSaveReqVO> documents, Long knowledgeBaseId, Integer chunkSize) {
private List<KnowledgeDocumentsDO> updateOrInsertDocuments (List<KnowledgeDocumentsSaveReqVO> documents, Long knowledgeBaseId) {
List<KnowledgeDocumentsDO> newDocuments = new ArrayList<>();
documents.forEach(doc -> {
KnowledgeDocumentsDO docDO = BeanUtils.toBean(doc, KnowledgeDocumentsDO.class);
docDO.setKnowledgeBaseId(knowledgeBaseId);
if (doc.getId() == null) {
newDocuments.add(docDO); // 收集新增文档
}
knowledgeDocumentsMapper.insertOrUpdate(docDO); // 更新或插入文档
// if (doc.getId() == null) {
newDocuments.add(docDO);
// }
docDO.setId(null);
knowledgeDocumentsMapper.insertOrUpdate(docDO);
});
return newDocuments;
}
@ -315,17 +297,23 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
}
Integer topK = 4;
if (baseDO.getTopK() == null || baseDO.getTopK() <= 0) {
testReqVO.setK(topK);
testReqVO.setTopK(topK);
if (testReqVO.getTopK() == null){
testReqVO.setTopK(baseDO.getTopK());
}
} else {
topK = baseDO.getTopK();
testReqVO.setK(topK);
topK = testReqVO.getTopK();
testReqVO.setTopK(topK);
if (testReqVO.getScore() == null){
testReqVO.setScore(baseDO.getScore());
}
}
Double score = 0.2;
if (baseDO.getScore() == null || baseDO.getTopK() <= 0.0 || baseDO.getScore() > 1) {
testReqVO.setScore(score);
} else {
score = baseDO.getScore();
score = testReqVO.getScore();
testReqVO.setScore(score);
}
@ -349,6 +337,39 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
return result;
}
/**
* 知识库Embedding
* @param request
*/
@Override
public void embeddingKnowledge (KnowledgeEmbeddingReqVO request) {
// 1. 校验知识库是否存在
validateKnowledgeParam(request);
// 2. 更新知识库主表基础信息
KnowledgeBaseDO updateObj = BeanUtils.toBean(request, KnowledgeBaseDO.class);
knowledgeBaseMapper.updateById(updateObj);
// 3. 处理附表（知识文档）数据
handleKnowledgeDocuments(request, updateObj);
}
/**
* 更新知识库命中测试参数
* @param updateReqVO
*/
@Override
public void updateHitParam (KnowledgeBaseSaveReqVO updateReqVO) {
// 校验存在
validateKnowledgeBaseExists(updateReqVO.getId());
// 更新
KnowledgeBaseDO updateObj = new KnowledgeBaseDO();
updateObj.setId(updateReqVO.getId())
.setTopK(updateReqVO.getTopK())
.setScore(updateReqVO.getScore())
;
knowledgeBaseMapper.updateById(updateObj);
}
/**
* 校验知识库是否存在