diff --git a/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java b/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java index 97f808bf5..ff9087a81 100644 --- a/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java +++ b/yudao-framework/yudao-common/src/main/java/cn/iocoder/yudao/framework/common/pojo/PageResult.java @@ -17,12 +17,6 @@ public final class PageResult implements Serializable { @Schema(description = "总量", requiredMode = Schema.RequiredMode.REQUIRED) private Long total; - @Schema(description = "页码") - private Integer pageNo; - - @Schema(description = "每页大小") - private Integer pageSize; - public PageResult() { } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java index dd03d1049..a272dcec6 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/KnowledgeBaseController.java @@ -107,7 +107,7 @@ public class KnowledgeBaseController { @GetMapping("/preview") @Operation(summary = "知识库文档预览") - public CommonResult> previewKnowledgeBaseDocument(@RequestBody KnowledgeBasePreviewReqVO reqVO) { + public CommonResult> previewKnowledgeBaseDocument(@Valid KnowledgeBasePreviewReqVO reqVO) { PageResult pageResult = knowledgeBaseService.previewKnowledgeBaseDocument(reqVO); return success(BeanUtils.toBean(pageResult,KnowledgeBasePreviewRespVO.class)); } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java index 338256888..165c588c4 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgebase/vo/KnowledgeBasePreviewRespVO.java @@ -11,6 +11,9 @@ import lombok.Data; @Data @Schema(description = "知识库文档预览响应信息") public class KnowledgeBasePreviewRespVO { + @Schema(description = "文档标号", example = "文档标号") + private Integer tit; + @Schema(description = "文件内容", example = "这是文件的内容") private String pageContent; diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java index 1ccbb96b3..6b6dffcd8 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/framework/backend/config/LLMBackendProperties.java @@ -123,6 +123,9 @@ public class LLMBackendProperties { private String deleteTheModel; + /** + * 知识库文件预览 + */ private String knowledgeBaseChunkedFiles; private String deleteTheModelFull; diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java index 74b8fe20e..be18ca84d 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.llm.service.async; import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewReqVO; +import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeBasePreviewRespVO; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestReqVO; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.KnowledgeHitRateTestResultVO; import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; @@ -10,9 +12,11 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.RagHttpService; import cn.iocoder.yudao.module.llm.service.http.vo.KnowledgeRagEmbedReqVO; +import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; import com.alibaba.fastjson.JSON; +import de.danielbechler.util.Collections; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +49,7 @@ public class AsyncKnowledgeBase { /** * 向向量知识库创建文件 + * * @param knowledgeList * @param knowledgeParameters */ @@ -87,8 +92,8 @@ public class AsyncKnowledgeBase { log.info("createKnowledgeBase 方法执行完成。"); } - public void deletedKnowledgeBase ( List ids) { - log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); + public void deletedKnowledgeBase (List ids) { + log.info("开始执行 deletedKnowledgeBase 方法。 ids 大小: {} ,ids:{} ", ids.size(), JSON.toJSON(ids)); // 如果提供了 ids，则删除现有的知识库文档 if (!CollectionUtils.isAnyEmpty(ids)) { @@ -105,6 +110,7 @@ public class AsyncKnowledgeBase { } } + /** * 修改知识库文档状态 * @@ -141,7 +147,7 @@ public class AsyncKnowledgeBase { } - public List executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO , List fileIds) { + public List executeHitRateTest (KnowledgeHitRateTestReqVO testReqVO, List fileIds) { List fileIdStr = fileIds.stream() .map(Object::toString) .collect(Collectors.toList()); @@ -168,9 +174,9 @@ public class AsyncKnowledgeBase { // 根据 fileId 查找文件名 KnowledgeDocumentsDO documents = knowledgeDocumentsMapper.selectOne(KnowledgeDocumentsDO::getFileId, fileId); - if (documents!=null && StringUtils.isNotBlank(documents.getDocumentName())){ + if (documents != null && StringUtils.isNotBlank(documents.getDocumentName())) { resultVO.setFileName(documents.getDocumentName()); - }else { + } else { resultVO.setFileName("未知文件"); } @@ -178,4 +184,29 @@ public class AsyncKnowledgeBase { } return resultList; } + + /** + * 知识库文档预览 + * + * @param reqVO + * @return + */ + public List previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) { + List result = ragHttpService.previewKnowledgeBaseDocument(reqVO.getFileId()); + if (Collections.isEmpty(result)) { + return new ArrayList<>(); + } + + int i = 1; + List pages = new ArrayList<>(); + for (PageContentInfoVO pageContentInfoVO : result) { + KnowledgeBasePreviewRespVO respVO = new KnowledgeBasePreviewRespVO(); + respVO.setTit(i++); + respVO.setPageContent(pageContentInfoVO.getPageContent()); + respVO.setWordCount(pageContentInfoVO.getPageContent().length()); + pages.add(respVO); + } + + return pages; + } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java index 4ef269755..b31fc85a8 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java @@ -3,6 +3,7 @@ package cn.iocoder.yudao.module.llm.service.http; import cn.hutool.http.Header; import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpUtil; import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.util.http.HttpUtils; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; @@ -13,12 +14,10 @@ import cn.iocoder.yudao.module.llm.enums.KnowledgeStatusEnum; import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; import cn.iocoder.yudao.module.llm.service.http.vo.*; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.DocumentInfoVO; +import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.PageContentInfoVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryMultipleReqVO; import cn.iocoder.yudao.module.llm.service.http.vo.query.multiple.QueryResultPairVO; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONException; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.*; import com.baomidou.mybatisplus.core.toolkit.BeanUtils; import com.google.gson.JsonArray; import com.vladsch.flexmark.html.HtmlRenderer; @@ -53,10 +52,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.TimeUnit; import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; @@ -749,16 +745,16 @@ public class RagHttpService { .execute().body(); cn.hutool.core.lang.Console.log(result2); - log.info("请求参数: {}",jsonString); - log.info("请求结果: {}",JSON.toJSONString(result2)); - return parseHitRateTestResults(result2,vo.getScore()); + log.info("请求参数: {}", jsonString); + log.info("请求结果: {}", JSON.toJSONString(result2)); + return parseHitRateTestResults(result2, vo.getScore()); } private static List parseHitRateTestResults (String json, Double score) { - boolean array= json.trim().startsWith("["); + boolean array = json.trim().startsWith("["); // 先判断 JSON 是否是一个数组 - if (!array){ + if (!array) { // 判断是否存在 detail 字段 JSONObject jsonObject = JSON.parseObject(json); if (jsonObject.containsKey("detail")) { @@ -799,13 +795,13 @@ public class RagHttpService { } -// // 访问数据 -// for (QueryResultPairVO pair : results) { -// System.out.println("Page Content: " + pair.getDocument().getPageContent()); -// System.out.println("Hit Rate: " + pair.getHitRate()); -// System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId()); -// System.out.println("----------------------"); -// } + // // 访问数据 + // for (QueryResultPairVO pair : results) { + // System.out.println("Page Content: " + pair.getDocument().getPageContent()); + // System.out.println("Hit Rate: " + pair.getHitRate()); + // System.out.println("File ID: " + pair.getDocument().getMetadata().getFileId()); + // System.out.println("----------------------"); + // } return results; } @@ -832,4 +828,36 @@ public class RagHttpService { } + public List previewKnowledgeBaseDocument (Long fileId) { + String url = llmBackendProperties.getKnowledgeBaseChunkedFiles(); + + + //可以单独传入http参数，这样参数会自动做URL编码，拼接在URL中 + HashMap paramMap = new HashMap<>(); + paramMap.put("ids", fileId); + + String result3 = HttpUtil.get(url, paramMap); + log.info("预览知识库文档请求结果: {}", JSON.toJSONString(result3)); + return parsePreviewDocumentResults(result3); + } + + private List parsePreviewDocumentResults (String result) { + boolean array = result.trim().startsWith("["); + // 先判断 JSON 是否是一个数组 + + if (!array) { + // 判断是否存在 detail 字段 + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.containsKey("detail")) { + String detail = jsonObject.getString("detail"); + + if (detail.contains("No documents found for the given query")) { + throw exception(new ErrorCode(100_100_1, "预览文档失败，请检查文档或重新上传")); + } + return new ArrayList<>(); + } + } + + return JSON.parseObject(result, new TypeReference>() {}); + } } diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index 969d20601..990887b30 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -4,7 +4,6 @@ import cn.iocoder.yudao.framework.common.exception.ErrorCode; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; -import cn.iocoder.yudao.framework.common.util.object.PageUtils; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.module.llm.controller.admin.knowledgebase.vo.*; import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.KnowledgeDocumentsRespVO; @@ -18,7 +17,6 @@ import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase; import cn.iocoder.yudao.module.llm.utils.PageUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import lombok.extern.slf4j.Slf4j; -import org.ehcache.shadow.org.terracotta.offheapstore.paging.Page; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -384,25 +382,11 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { @Override public PageResult previewKnowledgeBaseDocument (KnowledgeBasePreviewReqVO reqVO) { // 模拟分页数据 - List pageData = new ArrayList<>(); + List pageData = asyncKnowledgeBase.previewKnowledgeBaseDocument(reqVO); - // 第一页数据 - KnowledgeBasePreviewRespVO page1 = new KnowledgeBasePreviewRespVO(); - page1.setPageContent("这是第一页的内容"); - page1.setWordCount(100); - pageData.add(page1); - - // 第二页数据 - KnowledgeBasePreviewRespVO page2 = new KnowledgeBasePreviewRespVO(); - page2.setPageContent("这是第二页的内容"); - page2.setWordCount(150); - pageData.add(page2); - - // 第三页数据 - KnowledgeBasePreviewRespVO page3 = new KnowledgeBasePreviewRespVO(); - page3.setPageContent("这是第三页的内容"); - page3.setWordCount(200); - pageData.add(page3); + if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isEmpty(pageData)){ + return PageResult.empty(); + } // 使用 PageUtil 进行分页处理 List pagedList = PageUtil.page(reqVO.getPageNo(), reqVO.getPageSize(), pageData); @@ -411,8 +395,6 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { PageResult pageResult = new PageResult<>(); pageResult.setList(pagedList); pageResult.setTotal((long) pageData.size()); - pageResult.setPageNo(reqVO.getPageNo()); - pageResult.setPageSize(reqVO.getPageSize()); log.info("知识库文档预览成功，返回分页结果：{}", pageResult); return pageResult;