DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

修改模型工厂上传文件格式

Browse Source
This commit is contained in:
limin 2025-01-15 17:41:21 +08:00
parent 16d3e78890
commit 8e54e83794
1 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataSetService.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.hutool.json.JSONObject;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
@ -7,6 +8,7 @@ import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.service.dataset.vo.AigcDatasetVo;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.AigcDatasetFileRespV0;
import cn.iocoder.yudao.module.llm.service.http.vo.ModelCompletionsReqVO;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
@ -32,26 +34,28 @@ public class AsyncDataSetService {
@Async
public void JsonFileWrite(DatasetDO datasetDO,List<DatasetQuestionRespVO> datasetQuestionList) {
List<AigcDatasetVo> aigcDatasetVoList = new ArrayList<>();
StringBuilder sb = new StringBuilder();
datasetQuestionList.forEach(dataSource -> {
AigcDatasetVo aigcDatasetVo = new AigcDatasetVo();
aigcDatasetVo.setInstruction(StringUtils.isNotBlank(dataSource.getSystem())?dataSource.getSystem():"");
aigcDatasetVo.setInput(StringUtils.isNotBlank(dataSource.getQuestion())?dataSource.getQuestion():"");
List<ModelCompletionsReqVO.ModelCompletionsMessage> messages = new ArrayList<>();
ModelCompletionsReqVO.ModelCompletionsMessage questionMessage = new ModelCompletionsReqVO.ModelCompletionsMessage();
questionMessage.setRole("user");
questionMessage.setContent(StringUtils.isNotBlank(dataSource.getQuestion())?dataSource.getQuestion():"");
messages.add(questionMessage);
ModelCompletionsReqVO.ModelCompletionsMessage answerMessage = new ModelCompletionsReqVO.ModelCompletionsMessage();
answerMessage.setRole("assistant");
if (!CollectionUtils.isAnyEmpty(dataSource.getDatasetAnswerRespVO())){
aigcDatasetVo.setOutput(StringUtils.isNotBlank(dataSource.getDatasetAnswerRespVO().get(0).getAnswer())?dataSource.getDatasetAnswerRespVO().get(0).getAnswer():"");
answerMessage.setContent(StringUtils.isNotBlank(dataSource.getDatasetAnswerRespVO().get(0).getAnswer())?dataSource.getDatasetAnswerRespVO().get(0).getAnswer():"");
}else {
aigcDatasetVo.setOutput("");
answerMessage.setContent("");
}
aigcDatasetVoList.add(aigcDatasetVo);
messages.add(answerMessage);
JSONObject jsonObject = new JSONObject();
jsonObject.put("messages", messages);
sb.append(jsonObject.toString()).append("\n");
});
ObjectMapper mapper = new ObjectMapper();
try {
StringBuilder sb = new StringBuilder();
for (AigcDatasetVo aigcDatasetVo : aigcDatasetVoList) {
String json = mapper.writeValueAsString(aigcDatasetVo);
sb.append(json).append("\n"); // 每个 JSON 对象后换行
}
InputStream inputStream = new ByteArrayInputStream(sb.toString().getBytes());
try (InputStream inputStream = new ByteArrayInputStream(sb.toString().getBytes())){
AigcDatasetFileRespV0 aigcDatasetFileRespV0 = trainHttpService.AigcUploadFile(new HashMap<>(), inputStream, datasetDO.getDatasetName() + datasetDO.getId() + ".json");
if (aigcDatasetFileRespV0 != null){
datasetMapper.setJobid(datasetDO.getId(),aigcDatasetFileRespV0.getFileId());