diff --git a/yudao-module-llm/yudao-module-llm-biz/pom.xml b/yudao-module-llm/yudao-module-llm-biz/pom.xml index 96174e44b..604e3f010 100644 --- a/yudao-module-llm/yudao-module-llm-biz/pom.xml +++ b/yudao-module-llm/yudao-module-llm-biz/pom.xml @@ -60,6 +60,16 @@ opencc4j 1.8.1 + + com.konghq + unirest-java + 3.13.6 + + + com.googlecode.juniversalchardet + juniversalchardet + 1.0.3 + diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java index 173ca8652..74c2a7a4d 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java @@ -13,7 +13,7 @@ public class KnowledgeDocumentsSaveReqVO { private Long id; @Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "18229") - @NotNull(message = "知识库ID不能为空") +// @NotNull(message = "知识库ID不能为空") private Long knowledgeBaseId; @Schema(description = "文档名称", example = "芋艿") diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java index 6de421c8a..78367b746 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java @@ -8,6 +8,7 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; import org.apache.ibatis.annotations.Mapper; import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.*; +import org.apache.ibatis.annotations.Select; /** * 知识库文档 Mapper @@ -27,4 +28,7 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX selectDeleteIds(Long id); + } \ No newline at end of file diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java new file mode 100644 index 000000000..808149f65 --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java @@ -0,0 +1,49 @@ +package cn.iocoder.yudao.module.llm.service.async; + +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; +import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties; +import cn.iocoder.yudao.module.llm.service.http.RagHttpService; +import cn.iocoder.yudao.module.llm.service.http.vo.RegUploadReqVO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.List; + +@Service +public class AsyncKnowledgeBase { + + + private static final Logger log = LoggerFactory.getLogger(AsyncKnowledgeBase.class); + @Resource + private RagHttpService ragHttpService; + @Resource + private LLMBackendProperties llmBackendProperties; + + // 向向量知识库创建文件 + @Async + public void createKnowledgeBase(List knowledgeList,List ids) { + if (!CollectionUtils.isAnyEmpty(ids)){ + String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids); + log.info("delete knowledge base info {}",mes); + } + if (!CollectionUtils.isAnyEmpty(knowledgeList)){ + knowledgeList.stream().forEach(knowledge -> { + try { + RegUploadReqVO regUploadReqVO = new RegUploadReqVO() + .setUrl(llmBackendProperties.getRagEmbed()) + .setFileId(String.valueOf(knowledge.getId())) + .setFileName(knowledge.getDocumentName()) + .setFileUrl(knowledge.getFileUrl()); + ragHttpService.embedUploadFile(regUploadReqVO); + }catch (Exception e){ + log.error("the creation of the knowledge base error {}",e.getMessage()); + } + + }); + } + } +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java index 74d47b40f..1e2911148 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java @@ -8,10 +8,22 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.toolkit.BeanUtils; +import com.google.gson.JsonArray; +import kong.unirest.HttpResponse; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.mozilla.universalchardet.UniversalDetector; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -50,6 +62,80 @@ public class RagHttpService { return ragEmbedRespVO; } + /** + * 向量知识库文档上传 + * @param ragUploadReqVO + * @return + * @throws UnirestException + * @throws IOException + */ + public RagEmbedRespVO embedUploadFile(RegUploadReqVO ragUploadReqVO) throws UnirestException, IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + RagEmbedRespVO ragEmbedRespVO = new RagEmbedRespVO(); + HttpGet request = new HttpGet(ragUploadReqVO.getFileUrl()); + try (CloseableHttpResponse response = httpClient.execute(request)) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + try (InputStream inputStream = entity.getContent(); + BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream)) { + bufferedInputStream.mark(Integer.MAX_VALUE); + String encoding = detectCharset(bufferedInputStream); + bufferedInputStream.reset(); // 重置流以便重新读取 + // 使用检测到的编码读取文件内容 + try (InputStreamReader reader = new InputStreamReader(bufferedInputStream, encoding); + BufferedReader bufferedReader = new BufferedReader(reader)) { + StringBuilder fileContentBuilder = new StringBuilder(); + String line; + while ((line = bufferedReader.readLine()) != null) { + fileContentBuilder.append(line).append(System.lineSeparator()); + } + String fileContent = fileContentBuilder.toString(); + byte[] utf8Bytes = fileContent.getBytes(StandardCharsets.UTF_8); + + // 上传文件 + HttpResponse uploadResponse = Unirest.post(ragUploadReqVO.getUrl()) + .field("file_id", ragUploadReqVO.getFileId()) + .field("file", new ByteArrayInputStream(utf8Bytes), ragUploadReqVO.getFileName()) // 使用文件名 "file.txt" 作为示例 + .asString(); + log.info("Response Body: " + uploadResponse.getBody()); + ragEmbedRespVO = JSON.parseObject(uploadResponse.getBody(), RagEmbedRespVO.class); + log.info(" ragEmbedRespVO:{}", ragEmbedRespVO); + + } + } + } + } + return ragEmbedRespVO; + } + + private static String detectCharset(InputStream inputStream) throws IOException { + byte[] buffer = new byte[4096]; + int nread; + UniversalDetector detector = new UniversalDetector(null); + BufferedInputStream bis = new BufferedInputStream(inputStream); + while ((nread = bis.read(buffer)) > 0 && !detector.isDone()) { + detector.handleData(buffer, 0, nread); + } + detector.dataEnd(); + String charset = detector.getDetectedCharset(); + detector.reset(); + return charset; + } + + public String ragDocumentsDel(String url, List documentIds){ + // 创建JSON数组 + JsonArray jsonArray = new JsonArray(); + for (Long id : documentIds) { + jsonArray.add(String.valueOf(id)); + } + // 发送DELETE请求 + HttpResponse response = Unirest.delete(url) + .header("Content-Type", "application/json") + .body(jsonArray.toString()) + .asString(); + // 返回响应体 + return response.getBody(); + } /** * 获取所有向量id */ diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/RegUploadReqVO.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/RegUploadReqVO.java new file mode 100644 index 000000000..7b39b8eb9 --- /dev/null +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/RegUploadReqVO.java @@ -0,0 +1,16 @@ +package cn.iocoder.yudao.module.llm.service.http.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class RegUploadReqVO { + private String url; + private String fileName; + private String fileId; + private String fileUrl; + +} diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java index f8df585ce..15a6ed235 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java @@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgebase.KnowledgeBaseDO; import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO; import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper; import cn.iocoder.yudao.module.llm.dal.mysql.knowledgedocuments.KnowledgeDocumentsMapper; +import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase; import org.springframework.stereotype.Service; import org.springframework.validation.annotation.Validated; @@ -20,6 +21,7 @@ import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionU import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.KNOWLEDGE_BASE_NOT_EXISTS; import javax.annotation.Resource; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -36,6 +38,8 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { private KnowledgeBaseMapper knowledgeBaseMapper; @Resource private KnowledgeDocumentsMapper knowledgeDocumentsMapper; + @Resource + private AsyncKnowledgeBase asyncKnowledgeBase; @Override public Long createKnowledgeBase(KnowledgeBaseSaveReqVO createReqVO) { @@ -53,9 +57,12 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { // 更新 KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class); knowledgeBaseMapper.updateById(updateObj); + List knowledgeDocumentsList = new ArrayList<>(); // 附表增加数据 if (!CollectionUtils.isAnyEmpty(updateReqVO.getKnowledgeDocuments())){ - List ids = updateReqVO.getKnowledgeDocuments().stream().map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList()); + List ids = updateReqVO.getKnowledgeDocuments().stream().filter( + knowledgeDocuments -> knowledgeDocuments.getId() != null + ).map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList()); if (!CollectionUtils.isAnyEmpty(ids)){ knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX() .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()) @@ -68,9 +75,21 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { knowledgeDocuments -> { KnowledgeDocumentsDO knowledgeDocumentsDO = BeanUtils.toBean(knowledgeDocuments, KnowledgeDocumentsDO.class); knowledgeDocumentsDO.setKnowledgeBaseId(updateReqVO.getId()); + if (knowledgeDocuments.getId() == null){ + knowledgeDocumentsList.add(knowledgeDocumentsDO); + } knowledgeDocumentsMapper.insertOrUpdate(knowledgeDocumentsDO); } ); + List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); + asyncKnowledgeBase.createKnowledgeBase(knowledgeDocumentsList,deleteIds); + } else { + knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX() + .eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())); + List deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId()); + if (!CollectionUtils.isAnyEmpty(deleteIds)){ + asyncKnowledgeBase.createKnowledgeBase(null,deleteIds); + } } }