DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'xnjz/master'

Browse Source
This commit is contained in:
zhangtao 2025-01-06 18:06:11 +08:00
commit 4515de2918
10 changed files with 211 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
yudao-module-llm/yudao-module-llm-biz/pom.xml
View File

@ -60,6 +60,16 @@
<artifactId>opencc4j</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>com.konghq</groupId>
<artifactId>unirest-java</artifactId>
<version>3.13.6</version>
</dependency>
<dependency>
<groupId>com.googlecode.juniversalchardet</groupId>
<artifactId>juniversalchardet</artifactId>
<version>1.0.3</version>
</dependency>
</dependencies>

2
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/controller/admin/knowledgedocuments/vo/KnowledgeDocumentsSaveReqVO.java
View File

@ -13,7 +13,7 @@ public class KnowledgeDocumentsSaveReqVO {
private Long id;
@Schema(description = "知识库ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "18229")
@NotNull(message = "知识库ID不能为空")
// @NotNull(message = "知识库ID不能为空")
private Long knowledgeBaseId;
@Schema(description = "文档名称", example = "芋艿")

4
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/dal/mysql/knowledgedocuments/KnowledgeDocumentsMapper.java
View File

@ -8,6 +8,7 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
import org.apache.ibatis.annotations.Mapper;
import cn.iocoder.yudao.module.llm.controller.admin.knowledgedocuments.vo.*;
import org.apache.ibatis.annotations.Select;
/**
* 知识库文档 Mapper
@ -27,4 +28,7 @@ public interface KnowledgeDocumentsMapper extends BaseMapperX<KnowledgeDocuments
.orderByDesc(KnowledgeDocumentsDO::getId));
}
@Select("SELECT id FROM llm_knowledge_documents WHERE knowledge_base_id = #{id} and deleted = 1")
List<Long> selectDeleteIds(Long id);
}

49
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncKnowledgeBase.java Normal file
View File

@ -0,0 +1,49 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.RagHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.RegUploadReqVO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
@Service
public class AsyncKnowledgeBase {
private static final Logger log = LoggerFactory.getLogger(AsyncKnowledgeBase.class);
@Resource
private RagHttpService ragHttpService;
@Resource
private LLMBackendProperties llmBackendProperties;
// 向向量知识库创建文件
@Async
public void createKnowledgeBase(List<KnowledgeDocumentsDO> knowledgeList,List<Long> ids) {
if (!CollectionUtils.isAnyEmpty(ids)){
String mes = ragHttpService.ragDocumentsDel(llmBackendProperties.getRagDocumentsDel(), ids);
log.info("delete knowledge base info {}",mes);
}
if (!CollectionUtils.isAnyEmpty(knowledgeList)){
knowledgeList.stream().forEach(knowledge -> {
try {
RegUploadReqVO regUploadReqVO = new RegUploadReqVO()
.setUrl(llmBackendProperties.getRagEmbed())
.setFileId(String.valueOf(knowledge.getId()))
.setFileName(knowledge.getDocumentName())
.setFileUrl(knowledge.getFileUrl());
ragHttpService.embedUploadFile(regUploadReqVO);
}catch (Exception e){
log.error("the creation of the knowledge base error {}",e.getMessage());
}
});
}
}
}

23
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/basemodel/BaseModelTaskService.java
View File

@ -1,6 +1,5 @@
package cn.iocoder.yudao.module.llm.service.basemodel;
import cn.hutool.json.JSONUtil;
import cn.iocoder.yudao.module.llm.controller.admin.basemodel.vo.BaseModelSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.service.basemodel.vo.ModelListRes;
@ -8,16 +7,14 @@ import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.esotericsoftware.minlog.Log;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
@SpringBootApplication
@EnableScheduling
@Component
public class BaseModelTaskService {
@Resource
@ -64,5 +61,19 @@ public class BaseModelTaskService {
baseModelService.updateBaseModel(baseModelSaveReqVO);
}
}
List<BaseModelDO> baseModelList = baseModelService.getBaseModelList();
List<Long> ids = modelListRes.stream().map(ModelListRes::getId).collect(Collectors.toList());
for (BaseModelDO baseModelDO : baseModelList) {
if(baseModelDO.getAigcId() == null){
continue;
}
if(!ids.contains(baseModelDO.getAigcId())){
baseModelService.deleteBaseModel(baseModelDO.getId());
}
}
}
}

6
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/dataset/DatasetServiceImpl.java
View File

@ -87,6 +87,9 @@ public class DatasetServiceImpl implements DatasetService {
dataset.setAnnotateProgress(formattedRatio);
}
dataset.setStatus(status);
if(annoCount == 0){
dataset.setStatus(0);
}
datasetMapper.updateById(dataset);
}
return dataset.getId();
@ -132,6 +135,9 @@ public class DatasetServiceImpl implements DatasetService {
updateObj.setAnnotateProgress(formattedRatio);
}
updateObj.setStatus(status);
if(annoCount == 0){
updateObj.setStatus(0);
}
}
datasetMapper.updateById(updateObj);
}

86
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/RagHttpService.java
View File

@ -8,10 +8,22 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.toolkit.BeanUtils;
import com.google.gson.JsonArray;
import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
import kong.unirest.UnirestException;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.mozilla.universalchardet.UniversalDetector;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@ -50,6 +62,80 @@ public class RagHttpService {
return ragEmbedRespVO;
}
/**
* 向量知识库文档上传
* @param ragUploadReqVO
* @return
* @throws UnirestException
* @throws IOException
*/
public RagEmbedRespVO embedUploadFile(RegUploadReqVO ragUploadReqVO) throws UnirestException, IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
RagEmbedRespVO ragEmbedRespVO = new RagEmbedRespVO();
HttpGet request = new HttpGet(ragUploadReqVO.getFileUrl());
try (CloseableHttpResponse response = httpClient.execute(request)) {
HttpEntity entity = response.getEntity();
if (entity != null) {
try (InputStream inputStream = entity.getContent();
BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream)) {
bufferedInputStream.mark(Integer.MAX_VALUE);
String encoding = detectCharset(bufferedInputStream);
bufferedInputStream.reset(); // 重置流以便重新读取
// 使用检测到的编码读取文件内容
try (InputStreamReader reader = new InputStreamReader(bufferedInputStream, encoding);
BufferedReader bufferedReader = new BufferedReader(reader)) {
StringBuilder fileContentBuilder = new StringBuilder();
String line;
while ((line = bufferedReader.readLine()) != null) {
fileContentBuilder.append(line).append(System.lineSeparator());
}
String fileContent = fileContentBuilder.toString();
byte[] utf8Bytes = fileContent.getBytes(StandardCharsets.UTF_8);
// 上传文件
HttpResponse<String> uploadResponse = Unirest.post(ragUploadReqVO.getUrl())
.field("file_id", ragUploadReqVO.getFileId())
.field("file", new ByteArrayInputStream(utf8Bytes), ragUploadReqVO.getFileName()) // 使用文件名 "file.txt" 作为示例
.asString();
log.info("Response Body: " + uploadResponse.getBody());
ragEmbedRespVO = JSON.parseObject(uploadResponse.getBody(), RagEmbedRespVO.class);
log.info(" ragEmbedRespVO:{}", ragEmbedRespVO);
}
}
}
}
return ragEmbedRespVO;
}
private static String detectCharset(InputStream inputStream) throws IOException {
byte[] buffer = new byte[4096];
int nread;
UniversalDetector detector = new UniversalDetector(null);
BufferedInputStream bis = new BufferedInputStream(inputStream);
while ((nread = bis.read(buffer)) > 0 && !detector.isDone()) {
detector.handleData(buffer, 0, nread);
}
detector.dataEnd();
String charset = detector.getDetectedCharset();
detector.reset();
return charset;
}
public String ragDocumentsDel(String url, List<Long> documentIds){
// 创建JSON数组
JsonArray jsonArray = new JsonArray();
for (Long id : documentIds) {
jsonArray.add(String.valueOf(id));
}
// 发送DELETE请求
HttpResponse<String> response = Unirest.delete(url)
.header("Content-Type", "application/json")
.body(jsonArray.toString())
.asString();
// 返回响应体
return response.getBody();
}
/**
* 获取所有向量id
*/

16
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/http/vo/RegUploadReqVO.java Normal file
View File

@ -0,0 +1,16 @@
package cn.iocoder.yudao.module.llm.service.http.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class RegUploadReqVO {
private String url;
private String fileName;
private String fileId;
private String fileUrl;
}

21
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/knowledgebase/KnowledgeBaseServiceImpl.java
View File

@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgebase.KnowledgeBaseDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.knowledgedocuments.KnowledgeDocumentsDO;
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgebase.KnowledgeBaseMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.knowledgedocuments.KnowledgeDocumentsMapper;
import cn.iocoder.yudao.module.llm.service.async.AsyncKnowledgeBase;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -20,6 +21,7 @@ import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionU
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.KNOWLEDGE_BASE_NOT_EXISTS;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@ -36,6 +38,8 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
private KnowledgeBaseMapper knowledgeBaseMapper;
@Resource
private KnowledgeDocumentsMapper knowledgeDocumentsMapper;
@Resource
private AsyncKnowledgeBase asyncKnowledgeBase;
@Override
public Long createKnowledgeBase(KnowledgeBaseSaveReqVO createReqVO) {
@ -53,9 +57,12 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
// 更新
KnowledgeBaseDO updateObj = BeanUtils.toBean(updateReqVO, KnowledgeBaseDO.class);
knowledgeBaseMapper.updateById(updateObj);
List<KnowledgeDocumentsDO> knowledgeDocumentsList = new ArrayList<>();
// 附表增加数据
if (!CollectionUtils.isAnyEmpty(updateReqVO.getKnowledgeDocuments())){
List<Long> ids = updateReqVO.getKnowledgeDocuments().stream().map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList());
List<Long> ids = updateReqVO.getKnowledgeDocuments().stream().filter(
knowledgeDocuments -> knowledgeDocuments.getId() != null
).map(KnowledgeDocumentsSaveReqVO::getId).collect(Collectors.toList());
if (!CollectionUtils.isAnyEmpty(ids)){
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId())
@ -68,9 +75,21 @@ public class KnowledgeBaseServiceImpl implements KnowledgeBaseService {
knowledgeDocuments -> {
KnowledgeDocumentsDO knowledgeDocumentsDO = BeanUtils.toBean(knowledgeDocuments, KnowledgeDocumentsDO.class);
knowledgeDocumentsDO.setKnowledgeBaseId(updateReqVO.getId());
if (knowledgeDocuments.getId() == null){
knowledgeDocumentsList.add(knowledgeDocumentsDO);
}
knowledgeDocumentsMapper.insertOrUpdate(knowledgeDocumentsDO);
}
);
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
asyncKnowledgeBase.createKnowledgeBase(knowledgeDocumentsList,deleteIds);
} else {
knowledgeDocumentsMapper.delete(new LambdaQueryWrapperX<KnowledgeDocumentsDO>()
.eq(KnowledgeDocumentsDO::getKnowledgeBaseId, updateReqVO.getId()));
List<Long> deleteIds = knowledgeDocumentsMapper.selectDeleteIds(updateReqVO.getId());
if (!CollectionUtils.isAnyEmpty(deleteIds)){
asyncKnowledgeBase.createKnowledgeBase(null,deleteIds);
}
}
}

2
yudao-server/src/main/java/cn/iocoder/yudao/server/YudaoServerApplication.java
View File

@ -3,6 +3,7 @@ package cn.iocoder.yudao.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* 项目的启动类
@ -14,6 +15,7 @@ import org.springframework.scheduling.annotation.EnableAsync;
* @author 芋道源码
*/
@SuppressWarnings("SpringComponentScan") // 忽略 IDEA 无法识别 ${yudao.info.base-package}
@EnableScheduling
@SpringBootApplication(scanBasePackages = {"${yudao.info.base-package}.server", "${yudao.info.base-package}.module"})
public class YudaoServerApplication {