DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
sunxiqing 2025-03-15 17:35:36 +08:00
commit c3106010db
12 changed files with 131 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java
View File

@ -17,12 +17,6 @@ public final class PageResult<T> implements Serializable {
@Schema(description = "总量", requiredMode = Schema.RequiredMode.REQUIRED)
private Long total;
@Schema(description = "页码")
private Integer pageNo;
@Schema(description = "每页大小")
private Integer pageSize;
public PageResult() {
}

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java
View File

@ -107,7 +107,7 @@ public class KnowledgeBaseController {
@GetMapping("/preview")
@Operation(summary = "知识库文档预览")
public CommonResult<PageResult<KnowledgeBasePreviewRespVO>> previewKnowledgeBaseDocument(@RequestBody KnowledgeBasePreviewReqVO reqVO) {
public CommonResult<PageResult<KnowledgeBasePreviewRespVO>> previewKnowledgeBaseDocument(@Valid KnowledgeBasePreviewReqVO reqVO) {
PageResult<KnowledgeBasePreviewRespVO> pageResult = knowledgeBaseService.previewKnowledgeBaseDocument(reqVO);
return success(BeanUtils.toBean(pageResult,KnowledgeBasePreviewRespVO.class));
}

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java
View File

@ -11,6 +11,9 @@ import lombok.Data;
@Data
@Schema(description = "知识库文档预览响应信息")
public class KnowledgeBasePreviewRespVO {
@Schema(description = "文档标号", example = "文档标号")
private Integer tit;
@Schema(description = "文件内容", example = "这是文件的内容")
private String pageContent;

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java
View File

@ -123,6 +123,9 @@ public class LLMBackendProperties {
private String deleteTheModel;
/**
* 知识库文件预览
*/
private String knowledgeBaseChunkedFiles;
private String deleteTheModelFull;

44
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java
View File

@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestResultVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
@ -10,9 +12,11 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.RagHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO;
import com.alibaba.fastjson.JSON;
import de.danielbechler.util.Collections;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -45,6 +49,7 @@ public class AsyncKnowledgeBase {
/**
* 向向量知识库创建文件
*
* @param knowledgeList
* @param knowledgeParameters
*/
@ -87,8 +92,8 @@ public class AsyncKnowledgeBase {
log.info("createKnowledgeBase 方法执行完成。");
}
public void deletedKnowledgeBase ( List<Long> ids) {
log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids));
public void deletedKnowledgeBase (List<Long> ids) {
log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids));
// 如果提供了 ids，则删除现有的知识库文档
if (!CollectionUtils.isAnyEmpty(ids)) {
@ -105,6 +110,7 @@ public class AsyncKnowledgeBase {
}
}
/**
* 修改知识库文档状态
*
@ -141,7 +147,7 @@ public class AsyncKnowledgeBase {
}
public List<KnowledgeHitRateTestResultVO> executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO , List<Long> fileIds) {
public List<KnowledgeHitRateTestResultVO> executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO, List<Long> fileIds) {
List<String> fileIdStr = fileIds.stream()
.map(Object::toString)
.collect(Collectors.toList());
@ -158,19 +164,20 @@ public class AsyncKnowledgeBase {
KnowledgeHitRateTestResultVO resultVO = new KnowledgeHitRateTestResultVO();
resultVO.setPageContent(pair.getDocument().getPageContent());
DecimalFormat df = new DecimalFormat("0.00");
DecimalFormat df = new DecimalFormat("0.00%");
df.setRoundingMode(RoundingMode.HALF_UP);
String rateResult = df.format(pair.getHitRate());
resultVO.setHitRate(rateResult);
resultVO.setDigest(pair.getDocument().getMetadata().getDigest());
long fileId = Long.parseLong(pair.getDocument().getMetadata().getFileId());
resultVO.setFileId(fileId);
// 根据 fileId 查找文件名
KnowledgeDocumentsDO documents = knowledgeDocumentsMapper.selectOne(KnowledgeDocumentsDO::getFileId, fileId);
if (documents!=null && StringUtils.isNotBlank(documents.getDocumentName())){
if (documents != null && StringUtils.isNotBlank(documents.getDocumentName())) {
resultVO.setFileName(documents.getDocumentName());
}else {
} else {
resultVO.setFileName("未知文件");
}
@ -178,4 +185,29 @@ public class AsyncKnowledgeBase {
}
return resultList;
}
/**
* 知识库文档预览
*
* @param reqVO
* @return
*/
public List<KnowledgeBasePreviewRespVO> previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) {
List<PageContentInfoVO> result = ragHttpService.previewKnowledgeBaseDocument(reqVO.getFileId());
if (Collections.isEmpty(result)) {
return new ArrayList<>();
}
int i = 1;
List<KnowledgeBasePreviewRespVO> pages = new ArrayList<>();
for (PageContentInfoVO pageContentInfoVO : result) {
KnowledgeBasePreviewRespVO respVO = new KnowledgeBasePreviewRespVO();
respVO.setTit(i++);
respVO.setPageContent(pageContentInfoVO.getPageContent());
respVO.setWordCount(pageContentInfoVO.getPageContent().length());
pages.add(respVO);
}
return pages;
}
}

68
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java
View File

@ -3,6 +3,7 @@ package cn.iocoder.yudao.module.llm.service.http;
import cn.hutool.http.Header;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpUtil;
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
@ -13,12 +14,10 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.vo.*;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.DocumentInfoVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.*;
import com.baomidou.mybatisplus.core.toolkit.BeanUtils;
import com.google.gson.JsonArray;
import com.vladsch.flexmark.html.HtmlRenderer;
@ -53,10 +52,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
@ -749,16 +745,16 @@ public class RagHttpService {
.execute().body();
cn.hutool.core.lang.Console.log(result2);
log.info("请求参数: {}",jsonString);
log.info("请求结果: {}",JSON.toJSONString(result2));
return parseHitRateTestResults(result2,vo.getScore());
log.info("请求参数: {}", jsonString);
log.info("请求结果: {}", JSON.toJSONString(result2));
return parseHitRateTestResults(result2, vo.getScore());
}
private static List<QueryResultPairVO> parseHitRateTestResults (String json, Double score) {
boolean array= json.trim().startsWith("[");
boolean array = json.trim().startsWith("[");
// 先判断 JSON 是否是一个数组
if (!array){
if (!array) {
// 判断是否存在 detail 字段
JSONObject jsonObject = JSON.parseObject(json);
if (jsonObject.containsKey("detail")) {
@ -799,13 +795,13 @@ public class RagHttpService {
}
// // 访问数据
// for (QueryResultPairVO pair : results) {
// System.out.println("Page Content: " + pair.getDocument().getPageContent());
// System.out.println("Hit Rate: " + pair.getHitRate());
// System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId());
// System.out.println("----------------------");
// }
// // 访问数据
// for (QueryResultPairVO pair : results) {
// System.out.println("Page Content: " + pair.getDocument().getPageContent());
// System.out.println("Hit Rate: " + pair.getHitRate());
// System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId());
// System.out.println("----------------------");
// }
return results;
}
@ -832,4 +828,36 @@ public class RagHttpService {
}
public List<PageContentInfoVO> previewKnowledgeBaseDocument (Long fileId) {
String url = llmBackendProperties.getKnowledgeBaseChunkedFiles();
//可以单独传入http参数，这样参数会自动做URL编码，拼接在URL中
HashMap<String, Object> paramMap = new HashMap<>();
paramMap.put("ids", fileId);
String result3 = HttpUtil.get(url, paramMap);
log.info("预览知识库文档请求结果: {}", JSON.toJSONString(result3));
return parsePreviewDocumentResults(result3);
}
private List<PageContentInfoVO> parsePreviewDocumentResults (String result) {
boolean array = result.trim().startsWith("[");
// 先判断 JSON 是否是一个数组
if (!array) {
// 判断是否存在 detail 字段
JSONObject jsonObject = JSON.parseObject(result);
if (jsonObject.containsKey("detail")) {
String detail = jsonObject.getString("detail");
if (detail.contains("No documents found for the given query")) {
throw exception(new ErrorCode(100_100_1, "预览文档失败，请检查文档或重新上传"));
}
return new ArrayList<>();
}
}
return JSON.parseObject(result, new TypeReference<List<PageContentInfoVO>>() {});
}
}

11
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/query/multiple/PageContentInfoVO.java Normal file
View File

@ -0,0 +1,11 @@
package cn.iocoder.yudao.module.llm.service.http.vo.query.multiple;
import lombok.Data;
/**
* @Description 文档预览内容
*/
@Data
public class PageContentInfoVO {
String pageContent;
}

26
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java
View File

@ -4,7 +4,6 @@ import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.common.util.object.PageUtils;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.*;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsRespVO;
@ -18,7 +17,6 @@ import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase;
import cn.iocoder.yudao.module.llm.utils.PageUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import lombok.extern.slf4j.Slf4j;
import org.ehcache.shadow.org.terracotta.offheapstore.paging.Page;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -384,25 +382,11 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
@Override
public PageResult<KnowledgeBasePreviewRespVO> previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) {
// 模拟分页数据
List<KnowledgeBasePreviewRespVO> pageData = new ArrayList<>();
List<KnowledgeBasePreviewRespVO> pageData = asyncKnowledgeBase.previewKnowledgeBaseDocument(reqVO);
// 第一页数据
KnowledgeBasePreviewRespVO page1 = new KnowledgeBasePreviewRespVO();
page1.setPageContent("这是第一页的内容");
page1.setWordCount(100);
pageData.add(page1);
// 第二页数据
KnowledgeBasePreviewRespVO page2 = new KnowledgeBasePreviewRespVO();
page2.setPageContent("这是第二页的内容");
page2.setWordCount(150);
pageData.add(page2);
// 第三页数据
KnowledgeBasePreviewRespVO page3 = new KnowledgeBasePreviewRespVO();
page3.setPageContent("这是第三页的内容");
page3.setWordCount(200);
pageData.add(page3);
if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isEmpty(pageData)){
return PageResult.empty();
}
// 使用 PageUtil 进行分页处理
List<KnowledgeBasePreviewRespVO> pagedList = PageUtil.page(reqVO.getPageNo(), reqVO.getPageSize(), pageData);
@ -411,8 +395,6 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
PageResult<KnowledgeBasePreviewRespVO> pageResult = new PageResult<>();
pageResult.setList(pagedList);
pageResult.setTotal((long) pageData.size());
pageResult.setPageNo(reqVO.getPageNo());
pageResult.setPageSize(reqVO.getPageSize());
log.info("知识库文档预览成功，返回分页结果：{}", pageResult);
return pageResult;

8
yudao-server/src/main/resources/application-dev.yaml
View File

@ -291,7 +291,13 @@ llm:
deploy_model: http://36.103.199.248:5123/llm/deploy
# 模型删除
delete_model: http://36.103.199.248:5123/llm/deploy/stop
# 模型列表 get
a_list_of_available_models: http://36.103.199.248:5123/llm/list
#删除模型
delete_the_model: /delete_model
delete_the_model_full: http://36.103.199.248:5123/delete_model
# 知识库分块文件
knowledgeBaseChunkedFiles : http:/36.103.199.248:8123/documents
#################### 30000: 大模型对话相关API。 ###################
#### 大模型对话

4
yudao-server/src/main/resources/application-local.yaml
View File

@ -338,9 +338,9 @@ llm:
a_list_of_available_models: http://36.103.199.248:5123/llm/list
#删除模型
delete_the_model: /delete_model
delete_the_model_full: http://127.0.0.1:5123/delete_model
delete_the_model_full: http://36.103.199.248:5123/delete_model
# 知识库分块文件
knowledgeBaseChunkedFiles : /documents
knowledgeBaseChunkedFiles : http:/36.103.199.248:8123/documents
#################### 30000: 大模型对话相关API。 ###################

14
yudao-server/src/main/resources/application-prod.yaml
View File

@ -328,7 +328,19 @@ llm:
check_file_list: /llm/finetuning/checkpoints?model_name=
# 模型调优停止 POST
stop_finetuning: /llm/finetuning/stop
# 基座模型状态 POST
base_model_status: http://127.0.0.1:5123/llm/deploy/list
# 模型部署 POST
deploy_model: http://127.0.0.1:5123/llm/deploy
# 模型删除
delete_model: http://127.0.0.1:5123/llm/deploy/stop
# 模型列表 get
a_list_of_available_models: http://127.0.0.1:5123/llm/list
#删除模型
delete_the_model: /delete_model
delete_the_model_full: http://127.0.0.1:5123/delete_model
# 知识库分块文件
knowledgeBaseChunkedFiles: http://127.0.0.1:/documents
#################### 30000: 大模型对话相关API。 ###################
#### 大模型对话
# 模型列表 GET

2
yudao-server/src/main/resources/application-ymx.yaml
View File

@ -340,7 +340,7 @@ llm:
delete_the_model: /delete_model
delete_the_model_full: http://127.0.0.1:5123/delete_model
# 知识库分块文件
knowledgeBaseChunkedFiles: /documents
knowledgeBaseChunkedFiles: http://127.0.0.1:/documents
#################### 30000: 大模型对话相关API。 ###################
#### 大模型对话