diff --git a/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java b/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java index 97f808bf5..ff9087a81 100644 --- a/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java +++ b/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java @@ -17,12 +17,6 @@ public final class PageResult implements Serializable { @Schema(description = "总量", requiredMode = Schema.RequiredMode.REQUIRED) private Long total; - @Schema(description = "页码") - private Integer pageNo; - - @Schema(description = "每页大小") - private Integer pageSize; - public PageResult() { } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java index dd03d1049..a272dcec6 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java @@ -107,7 +107,7 @@ public class KnowledgeBaseController { @GetMapping("/preview") @Operation(summary = "知识库文档预览") - public CommonResult> previewKnowledgeBaseDocument(@RequestBody KnowledgeBasePreviewReqVO reqVO) { + public CommonResult> previewKnowledgeBaseDocument(@Valid KnowledgeBasePreviewReqVO reqVO) { PageResult pageResult = knowledgeBaseService.previewKnowledgeBaseDocument(reqVO); return success(BeanUtils.toBean(pageResult,KnowledgeBasePreviewRespVO.class)); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java index 338256888..165c588c4 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java @@ -11,6 +11,9 @@ import lombok.Data; @Data @Schema(description = "知识库文档预览响应信息") public class KnowledgeBasePreviewRespVO { + @Schema(description = "文档标号", example = "文档标号") + private Integer tit; + @Schema(description = "文件内容", example = "这是文件的内容") private String pageContent; diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java index 1ccbb96b3..6b6dffcd8 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java @@ -123,6 +123,9 @@ public class LLMBackendProperties { private String deleteTheModel; + /** + * 知识库文件预览 + */ private String knowledgeBaseChunkedFiles; private String deleteTheModelFull; diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 74b8fe20e..52042c558 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.llm.service.async; import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewReqVO; +import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewRespVO; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestReqVO; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestResultVO; import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; @@ -10,9 +12,11 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.RagHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; +import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; import com.alibaba.fastjson.JSON; +import de.danielbechler.util.Collections; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +49,7 @@ public class AsyncKnowledgeBase { /** * 向向量知识库创建文件 + * * @param knowledgeList * @param knowledgeParameters */ @@ -87,8 +92,8 @@ public class AsyncKnowledgeBase { log.info("createKnowledgeBase 方法执行完成。"); } - public void deletedKnowledgeBase ( List ids) { - log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); + public void deletedKnowledgeBase (List ids) { + log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); // 如果提供了 ids,则删除现有的知识库文档 if (!CollectionUtils.isAnyEmpty(ids)) { @@ -105,6 +110,7 @@ public class AsyncKnowledgeBase { } } + /** * 修改知识库文档状态 * @@ -141,7 +147,7 @@ public class AsyncKnowledgeBase { } - public List executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO , List fileIds) { + public List executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO, List fileIds) { List fileIdStr = fileIds.stream() .map(Object::toString) .collect(Collectors.toList()); @@ -158,19 +164,20 @@ public class AsyncKnowledgeBase { KnowledgeHitRateTestResultVO resultVO = new KnowledgeHitRateTestResultVO(); resultVO.setPageContent(pair.getDocument().getPageContent()); - DecimalFormat df = new DecimalFormat("0.00"); + DecimalFormat df = new DecimalFormat("0.00%"); df.setRoundingMode(RoundingMode.HALF_UP); String rateResult = df.format(pair.getHitRate()); resultVO.setHitRate(rateResult); + resultVO.setDigest(pair.getDocument().getMetadata().getDigest()); long fileId = Long.parseLong(pair.getDocument().getMetadata().getFileId()); resultVO.setFileId(fileId); // 根据 fileId 查找文件名 KnowledgeDocumentsDO documents = knowledgeDocumentsMapper.selectOne(KnowledgeDocumentsDO::getFileId, fileId); - if (documents!=null && StringUtils.isNotBlank(documents.getDocumentName())){ + if (documents != null && StringUtils.isNotBlank(documents.getDocumentName())) { resultVO.setFileName(documents.getDocumentName()); - }else { + } else { resultVO.setFileName("未知文件"); } @@ -178,4 +185,29 @@ public class AsyncKnowledgeBase { } return resultList; } + + /** + * 知识库文档预览 + * + * @param reqVO + * @return + */ + public List previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) { + List result = ragHttpService.previewKnowledgeBaseDocument(reqVO.getFileId()); + if (Collections.isEmpty(result)) { + return new ArrayList<>(); + } + + int i = 1; + List pages = new ArrayList<>(); + for (PageContentInfoVO pageContentInfoVO : result) { + KnowledgeBasePreviewRespVO respVO = new KnowledgeBasePreviewRespVO(); + respVO.setTit(i++); + respVO.setPageContent(pageContentInfoVO.getPageContent()); + respVO.setWordCount(pageContentInfoVO.getPageContent().length()); + pages.add(respVO); + } + + return pages; + } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java index 4ef269755..b31fc85a8 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java @@ -3,6 +3,7 @@ package cn.iocoder.yudao.module.llm.service.http; import cn.hutool.http.Header; import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpUtil; import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.util.http.HttpUtils; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; @@ -13,12 +14,10 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.vo.*; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.DocumentInfoVO; +import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONException; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.*; import com.baomidou.mybatisplus.core.toolkit.BeanUtils; import com.google.gson.JsonArray; import com.vladsch.flexmark.html.HtmlRenderer; @@ -53,10 +52,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.TimeUnit; import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; @@ -749,16 +745,16 @@ public class RagHttpService { .execute().body(); cn.hutool.core.lang.Console.log(result2); - log.info("请求参数: {}",jsonString); - log.info("请求结果: {}",JSON.toJSONString(result2)); - return parseHitRateTestResults(result2,vo.getScore()); + log.info("请求参数: {}", jsonString); + log.info("请求结果: {}", JSON.toJSONString(result2)); + return parseHitRateTestResults(result2, vo.getScore()); } private static List parseHitRateTestResults (String json, Double score) { - boolean array= json.trim().startsWith("["); + boolean array = json.trim().startsWith("["); // 先判断 JSON 是否是一个数组 - if (!array){ + if (!array) { // 判断是否存在 detail 字段 JSONObject jsonObject = JSON.parseObject(json); if (jsonObject.containsKey("detail")) { @@ -799,13 +795,13 @@ public class RagHttpService { } -// // 访问数据 -// for (QueryResultPairVO pair : results) { -// System.out.println("Page Content: " + pair.getDocument().getPageContent()); -// System.out.println("Hit Rate: " + pair.getHitRate()); -// System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId()); -// System.out.println("----------------------"); -// } + // // 访问数据 + // for (QueryResultPairVO pair : results) { + // System.out.println("Page Content: " + pair.getDocument().getPageContent()); + // System.out.println("Hit Rate: " + pair.getHitRate()); + // System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId()); + // System.out.println("----------------------"); + // } return results; } @@ -832,4 +828,36 @@ public class RagHttpService { } + public List previewKnowledgeBaseDocument (Long fileId) { + String url = llmBackendProperties.getKnowledgeBaseChunkedFiles(); + + + //可以单独传入http参数,这样参数会自动做URL编码,拼接在URL中 + HashMap paramMap = new HashMap<>(); + paramMap.put("ids", fileId); + + String result3 = HttpUtil.get(url, paramMap); + log.info("预览知识库文档请求结果: {}", JSON.toJSONString(result3)); + return parsePreviewDocumentResults(result3); + } + + private List parsePreviewDocumentResults (String result) { + boolean array = result.trim().startsWith("["); + // 先判断 JSON 是否是一个数组 + + if (!array) { + // 判断是否存在 detail 字段 + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.containsKey("detail")) { + String detail = jsonObject.getString("detail"); + + if (detail.contains("No documents found for the given query")) { + throw exception(new ErrorCode(100_100_1, "预览文档失败,请检查文档或重新上传")); + } + return new ArrayList<>(); + } + } + + return JSON.parseObject(result, new TypeReference>() {}); + } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/query/multiple/PageContentInfoVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/query/multiple/PageContentInfoVO.java new file mode 100644 index 000000000..0a02f6634 --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/query/multiple/PageContentInfoVO.java @@ -0,0 +1,11 @@ +package cn.iocoder.yudao.module.llm.service.http.vo.query.multiple; + +import lombok.Data; + +/** + * @Description 文档预览内容 + */ +@Data +public class PageContentInfoVO { + String pageContent; +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 969d20601..990887b30 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -4,7 +4,6 @@ import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; -import cn.iocoder.yudao.framework.common.util.object.PageUtils; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.*; import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsRespVO; @@ -18,7 +17,6 @@ import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase; import cn.iocoder.yudao.module.llm.utils.PageUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import lombok.extern.slf4j.Slf4j; -import org.ehcache.shadow.org.terracotta.offheapstore.paging.Page; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -384,25 +382,11 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { @Override public PageResult previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) { // 模拟分页数据 - List pageData = new ArrayList<>(); + List pageData = asyncKnowledgeBase.previewKnowledgeBaseDocument(reqVO); - // 第一页数据 - KnowledgeBasePreviewRespVO page1 = new KnowledgeBasePreviewRespVO(); - page1.setPageContent("这是第一页的内容"); - page1.setWordCount(100); - pageData.add(page1); - - // 第二页数据 - KnowledgeBasePreviewRespVO page2 = new KnowledgeBasePreviewRespVO(); - page2.setPageContent("这是第二页的内容"); - page2.setWordCount(150); - pageData.add(page2); - - // 第三页数据 - KnowledgeBasePreviewRespVO page3 = new KnowledgeBasePreviewRespVO(); - page3.setPageContent("这是第三页的内容"); - page3.setWordCount(200); - pageData.add(page3); + if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isEmpty(pageData)){ + return PageResult.empty(); + } // 使用 PageUtil 进行分页处理 List pagedList = PageUtil.page(reqVO.getPageNo(), reqVO.getPageSize(), pageData); @@ -411,8 +395,6 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { PageResult pageResult = new PageResult<>(); pageResult.setList(pagedList); pageResult.setTotal((long) pageData.size()); - pageResult.setPageNo(reqVO.getPageNo()); - pageResult.setPageSize(reqVO.getPageSize()); log.info("知识库文档预览成功,返回分页结果:{}", pageResult); return pageResult; diff --git a/yudao-server/src/main/resources/application-dev.yaml b/yudao-server/src/main/resources/application-dev.yaml index 7a010954e..f23db338d 100644 --- a/yudao-server/src/main/resources/application-dev.yaml +++ b/yudao-server/src/main/resources/application-dev.yaml @@ -291,7 +291,13 @@ llm: deploy_model: http://36.103.199.248:5123/llm/deploy # 模型删除 delete_model: http://36.103.199.248:5123/llm/deploy/stop - + # 模型列表 get + a_list_of_available_models: http://36.103.199.248:5123/llm/list + #删除模型 + delete_the_model: /delete_model + delete_the_model_full: http://36.103.199.248:5123/delete_model + # 知识库分块文件 + knowledgeBaseChunkedFiles : http:/36.103.199.248:8123/documents #################### 30000: 大模型对话相关API。 ################### #### 大模型对话 diff --git a/yudao-server/src/main/resources/application-local.yaml b/yudao-server/src/main/resources/application-local.yaml index e5c4c8daf..19b0fed7a 100644 --- a/yudao-server/src/main/resources/application-local.yaml +++ b/yudao-server/src/main/resources/application-local.yaml @@ -338,9 +338,9 @@ llm: a_list_of_available_models: http://36.103.199.248:5123/llm/list #删除模型 delete_the_model: /delete_model - delete_the_model_full: http://127.0.0.1:5123/delete_model + delete_the_model_full: http://36.103.199.248:5123/delete_model # 知识库分块文件 - knowledgeBaseChunkedFiles : /documents + knowledgeBaseChunkedFiles : http:/36.103.199.248:8123/documents #################### 30000: 大模型对话相关API。 ################### diff --git a/yudao-server/src/main/resources/application-prod.yaml b/yudao-server/src/main/resources/application-prod.yaml index 203311002..e17f9cba6 100644 --- a/yudao-server/src/main/resources/application-prod.yaml +++ b/yudao-server/src/main/resources/application-prod.yaml @@ -328,7 +328,19 @@ llm: check_file_list: /llm/finetuning/checkpoints?model_name= # 模型调优停止 POST stop_finetuning: /llm/finetuning/stop - + # 基座模型状态 POST + base_model_status: http://127.0.0.1:5123/llm/deploy/list + # 模型部署 POST + deploy_model: http://127.0.0.1:5123/llm/deploy + # 模型删除 + delete_model: http://127.0.0.1:5123/llm/deploy/stop + # 模型列表 get + a_list_of_available_models: http://127.0.0.1:5123/llm/list + #删除模型 + delete_the_model: /delete_model + delete_the_model_full: http://127.0.0.1:5123/delete_model + # 知识库分块文件 + knowledgeBaseChunkedFiles: http://127.0.0.1:/documents #################### 30000: 大模型对话相关API。 ################### #### 大模型对话 # 模型列表 GET diff --git a/yudao-server/src/main/resources/application-ymx.yaml b/yudao-server/src/main/resources/application-ymx.yaml index 1631a4ffc..ff0e52c4a 100644 --- a/yudao-server/src/main/resources/application-ymx.yaml +++ b/yudao-server/src/main/resources/application-ymx.yaml @@ -340,7 +340,7 @@ llm: delete_the_model: /delete_model delete_the_model_full: http://127.0.0.1:5123/delete_model # 知识库分块文件 - knowledgeBaseChunkedFiles: /documents + knowledgeBaseChunkedFiles: http://127.0.0.1:/documents #################### 30000: 大模型对话相关API。 ################### #### 大模型对话