This commit is contained in:
ire 2025-01-16 17:50:05 +08:00
parent fda24951d7
commit e80252e66b
20 changed files with 700 additions and 65 deletions

View File

@ -79,6 +79,7 @@ public class ModelAssessTaskAutoController {
return success(result);
}
@GetMapping("/page")
@Operation(summary = "获得模型评估任务 —— 自动评估分页")
// @PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto:query')")

View File

@ -0,0 +1,101 @@
package cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto;
import cn.iocoder.yudao.framework.apilog.core.annotation.ApiAccessLog;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.excel.core.util.ExcelUtils;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo.ModelAssessTaskAutoInfoDO;
import cn.iocoder.yudao.module.llm.service.modelassesstaskautoinfo.ModelAssessTaskAutoInfoService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import java.io.IOException;
import java.util.List;
import static cn.iocoder.yudao.framework.apilog.core.enums.OperateTypeEnum.EXPORT;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
@Tag(name = "管理后台 - 模型评估任务 —— 自动评估-详细内容")
@RestController
@RequestMapping("/llm/model-assess-task-auto-info")
@Validated
public class ModelAssessTaskAutoInfoController {
@Resource
private ModelAssessTaskAutoInfoService modelAssessTaskAutoInfoService;
@PostMapping("/create")
@Operation(summary = "创建模型评估任务 —— 自动评估-详细内容")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:create')")
public CommonResult<Long> createModelAssessTaskAutoInfo(@Valid @RequestBody ModelAssessTaskAutoInfoSaveReqVO createReqVO) {
return success(modelAssessTaskAutoInfoService.createModelAssessTaskAutoInfo(createReqVO));
}
@PutMapping("/update")
@Operation(summary = "更新模型评估任务 —— 自动评估-详细内容")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:update')")
public CommonResult<Boolean> updateModelAssessTaskAutoInfo(@Valid @RequestBody ModelAssessTaskAutoInfoSaveReqVO updateReqVO) {
modelAssessTaskAutoInfoService.updateModelAssessTaskAutoInfo(updateReqVO);
return success(true);
}
@DeleteMapping("/delete")
@Operation(summary = "删除模型评估任务 —— 自动评估-详细内容")
@Parameter(name = "id", description = "编号", required = true)
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:delete')")
public CommonResult<Boolean> deleteModelAssessTaskAutoInfo(@RequestParam("id") Long id) {
modelAssessTaskAutoInfoService.deleteModelAssessTaskAutoInfo(id);
return success(true);
}
@GetMapping("/get")
@Operation(summary = "获得模型评估任务 —— 自动评估-详细内容")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:query')")
public CommonResult<ModelAssessTaskAutoInfoRespVO> getModelAssessTaskAutoInfo(@RequestParam("id") Long id) {
ModelAssessTaskAutoInfoDO modelAssessTaskAutoInfo = modelAssessTaskAutoInfoService.getModelAssessTaskAutoInfo(id);
return success(BeanUtils.toBean(modelAssessTaskAutoInfo, ModelAssessTaskAutoInfoRespVO.class));
}
@GetMapping("/list")
@Operation(summary = "根绝任务id获得模型评估任务-自动评估")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:query')")
public CommonResult<List<ModelAssessTaskAutoInfoRespVO>> list(@Valid ModelAssessTaskAutoInfoPageReqVO pageReqVO) {
List<ModelAssessTaskAutoInfoDO> list = modelAssessTaskAutoInfoService.getListByTaskId(pageReqVO);
return success(BeanUtils.toBean(list, ModelAssessTaskAutoInfoRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "获得模型评估任务 —— 自动评估-详细内容分页")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:query')")
public CommonResult<PageResult<ModelAssessTaskAutoInfoRespVO>> getModelAssessTaskAutoInfoPage(@Valid ModelAssessTaskAutoInfoPageReqVO pageReqVO) {
PageResult<ModelAssessTaskAutoInfoDO> pageResult = modelAssessTaskAutoInfoService.getModelAssessTaskAutoInfoPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, ModelAssessTaskAutoInfoRespVO.class));
}
@GetMapping("/export-excel")
@Operation(summary = "导出模型评估任务 —— 自动评估-详细内容 Excel")
@PreAuthorize("@ss.hasPermission('llm:model-assess-task-auto-info:export')")
@ApiAccessLog(operateType = EXPORT)
public void exportModelAssessTaskAutoInfoExcel(@Valid ModelAssessTaskAutoInfoPageReqVO pageReqVO,
HttpServletResponse response) throws IOException {
pageReqVO.setPageSize(PageParam.PAGE_SIZE_NONE);
List<ModelAssessTaskAutoInfoDO> list = modelAssessTaskAutoInfoService.getModelAssessTaskAutoInfoPage(pageReqVO).getList();
// 导出 Excel
ExcelUtils.write(response, "模型评估任务 —— 自动评估-详细内容.xls", "数据", ModelAssessTaskAutoInfoRespVO.class,
BeanUtils.toBean(list, ModelAssessTaskAutoInfoRespVO.class));
}
}

View File

@ -0,0 +1,46 @@
package cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo;
import lombok.*;
import java.util.*;
import io.swagger.v3.oas.annotations.media.Schema;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import org.springframework.format.annotation.DateTimeFormat;
import java.time.LocalDateTime;
import static cn.iocoder.yudao.framework.common.util.date.DateUtils.FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND;
@Schema(description = "管理后台 - 模型评估任务 —— 自动评估-详细内容分页 Request VO")
@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class ModelAssessTaskAutoInfoPageReqVO extends PageParam {
@Schema(description = "自动评估任务id", example = "12558")
private Long taskId;
@Schema(description = "模型名称", example = "芋艿")
private String modelName;
@Schema(description = "问题")
private String question;
@Schema(description = "模型回答内容")
private String prompt;
@Schema(description = "数据集回答内容")
private String dataSetPrompt;
@Schema(description = "事实性错误")
private Integer factualError;
@Schema(description = "情感倾向性")
private Integer emotionTendency;
@Schema(description = "语义连贯性")
private Integer semanticsCoherence;
@Schema(description = "创建时间")
@DateTimeFormat(pattern = FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND)
private LocalDateTime[] createTime;
}

View File

@ -0,0 +1,55 @@
package cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
import org.springframework.format.annotation.DateTimeFormat;
import java.time.LocalDateTime;
import com.alibaba.excel.annotation.*;
@Schema(description = "管理后台 - 模型评估任务 —— 自动评估-详细内容 Response VO")
@Data
@ExcelIgnoreUnannotated
public class ModelAssessTaskAutoInfoRespVO {
@Schema(description = "评估任务ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "11736")
@ExcelProperty("评估任务ID")
private Long id;
@Schema(description = "自动评估任务id", example = "12558")
@ExcelProperty("自动评估任务id")
private Long taskId;
@Schema(description = "模型名称", example = "芋艿")
@ExcelProperty("模型名称")
private String modelName;
@Schema(description = "问题")
@ExcelProperty("问题")
private String question;
@Schema(description = "模型回答内容")
@ExcelProperty("模型回答内容")
private String prompt;
@Schema(description = "数据集回答内容")
@ExcelProperty("数据集回答内容")
private String dataSetPrompt;
@Schema(description = "事实性错误")
@ExcelProperty("事实性错误")
private Integer factualError;
@Schema(description = "情感倾向性")
@ExcelProperty("情感倾向性")
private Integer emotionTendency;
@Schema(description = "语义连贯性")
@ExcelProperty("语义连贯性")
private Integer semanticsCoherence;
@Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED)
@ExcelProperty("创建时间")
private LocalDateTime createTime;
}

View File

@ -0,0 +1,39 @@
package cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import java.util.*;
import javax.validation.constraints.*;
@Schema(description = "管理后台 - 模型评估任务 —— 自动评估-详细内容新增/修改 Request VO")
@Data
public class ModelAssessTaskAutoInfoSaveReqVO {
@Schema(description = "评估任务ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "11736")
private Long id;
@Schema(description = "自动评估任务id", example = "12558")
private Long taskId;
@Schema(description = "模型名称", example = "芋艿")
private String modelName;
@Schema(description = "问题")
private String question;
@Schema(description = "模型回答内容")
private String prompt;
@Schema(description = "数据集回答内容")
private String dataSetPrompt;
@Schema(description = "事实性错误")
private Integer factualError;
@Schema(description = "情感倾向性")
private Integer emotionTendency;
@Schema(description = "语义连贯性")
private Integer semanticsCoherence;
}

View File

@ -34,7 +34,7 @@ public class ModelAssessTaskAutoPageReqVO extends PageParam {
@DateTimeFormat(pattern = FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND)
private LocalDateTime[] createTime;
@Schema(description = "评估方向使用字典llm_dimension_type")
private Integer dimension;
@Schema(description = "评估方向")
private List<String> dimension;
}
}

View File

@ -44,9 +44,8 @@ public class ModelAssessTaskAutoRespVO {
@ExcelProperty("创建时间")
private LocalDateTime createTime;
@Schema(description = "评估方向使用字典llm_dimension_type")
@ExcelProperty("评估方向使用字典llm_dimension_type")
private Integer dimension;
@Schema(description = "评估方向")
private List<String> dimension;
@Schema(description = "自动评估列表")
private List<ModelAssessTaskStoplistRespVO> stoplists;

View File

@ -31,8 +31,8 @@ public class ModelAssessTaskAutoSaveReqVO {
@Schema(description = "是否预知模型服务0普通 1官方",example = "0")
private Integer modelType;
@Schema(description = "评估方向使用字典llm_dimension_type")
private Integer dimension;
@Schema(description = "评估方向")
private List<String> dimension;
@Schema(description = "自动规则打分,停用词表")
private List<ModelAssessTaskStoplistSaveReqVO> stoplists;

View File

@ -59,7 +59,7 @@ public class ModelAssessTaskAutoDO extends BaseDO {
/**
* 评估方向使用字典llm_dimension_type
*/
private Integer dimension;
private String dimension;
/**
* 状态
*/

View File

@ -0,0 +1,63 @@
package cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo;
import lombok.*;
import java.util.*;
import java.time.LocalDateTime;
import java.time.LocalDateTime;
import com.baomidou.mybatisplus.annotation.*;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
/**
* 模型评估任务 自动评估-详细内容 DO
*
* @author 华大大模型
*/
@TableName("llm_model_assess_task_auto_info")
@KeySequence("llm_model_assess_task_auto_info_seq") // 用于 OraclePostgreSQLKingbaseDB2H2 数据库的主键自增如果是 MySQL 等数据库可不写
@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ModelAssessTaskAutoInfoDO extends BaseDO {
/**
* 评估任务ID
*/
@TableId
private Long id;
/**
* 自动评估任务id
*/
private Long taskId;
/**
* 模型名称
*/
private String modelName;
/**
* 问题
*/
private String question;
/**
* 模型回答内容
*/
private String prompt;
/**
* 数据集回答内容
*/
private String dataSetPrompt;
/**
* 事实性错误
*/
private Integer factualError;
/**
* 情感倾向性
*/
private Integer emotionTendency;
/**
* 语义连贯性
*/
private Integer semanticsCoherence;
}

View File

@ -0,0 +1,32 @@
package cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskautoinfo;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoPageReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo.ModelAssessTaskAutoInfoDO;
import org.apache.ibatis.annotations.Mapper;
/**
* 模型评估任务 自动评估-详细内容 Mapper
*
* @author 华大大模型
*/
@Mapper
public interface ModelAssessTaskAutoInfoMapper extends BaseMapperX<ModelAssessTaskAutoInfoDO> {
default PageResult<ModelAssessTaskAutoInfoDO> selectPage(ModelAssessTaskAutoInfoPageReqVO reqVO) {
return selectPage(reqVO, new LambdaQueryWrapperX<ModelAssessTaskAutoInfoDO>()
.eqIfPresent(ModelAssessTaskAutoInfoDO::getTaskId, reqVO.getTaskId())
.likeIfPresent(ModelAssessTaskAutoInfoDO::getModelName, reqVO.getModelName())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getQuestion, reqVO.getQuestion())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getPrompt, reqVO.getPrompt())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getDataSetPrompt, reqVO.getDataSetPrompt())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getFactualError, reqVO.getFactualError())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getEmotionTendency, reqVO.getEmotionTendency())
.eqIfPresent(ModelAssessTaskAutoInfoDO::getSemanticsCoherence, reqVO.getSemanticsCoherence())
.betweenIfPresent(ModelAssessTaskAutoInfoDO::getCreateTime, reqVO.getCreateTime())
.orderByDesc(ModelAssessTaskAutoInfoDO::getId));
}
}

View File

@ -94,5 +94,8 @@ public class LLMBackendProperties {
private String optimizePrompt;
private String aigcModelCompletions;
private String aigcFileUpload;
private String autoEvaluation;
}

View File

@ -2,27 +2,38 @@ package cn.iocoder.yudao.module.llm.service.async;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetAnswerRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskManualSaveReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanualbackup.vo.ModelAssessTaskManualBackupSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskauto.ModelAssessTaskAutoDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo.ModelAssessTaskAutoInfoDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanual.ManualModelAnswerDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanual.ModelAssessTaskDimensionDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanual.ModelAssessTaskManualDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.ManualModelAnswerBackupDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.ModelAssessTaskDimensionBackupDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskmanualbackup.ModelAssessTaskManualBackupDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelservice.ModelServiceDO;
import cn.iocoder.yudao.module.llm.dal.mysql.basemodel.BaseModelMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.dataset.DatasetMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskauto.ModelAssessTaskAutoMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskautoinfo.ModelAssessTaskAutoInfoMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanual.ManualModelAnswerMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanual.ModelAssessTaskDimensionMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanual.ModelAssessTaskManualMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanualbackup.ManualModelAnswerBackupMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskmanualbackup.ModelAssessTaskDimensionBackupMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelservice.ModelServiceMapper;
import cn.iocoder.yudao.module.llm.service.conversation.ConversationService;
import cn.iocoder.yudao.module.llm.service.dataset.DatasetQuestionService;
import cn.iocoder.yudao.module.llm.service.http.ModelService;
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
import cn.iocoder.yudao.module.llm.service.http.vo.ModelCompletionsReqVO;
import cn.iocoder.yudao.module.llm.service.http.vo.ModelCompletionsRespVO;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.hibernate.validator.internal.engine.constraintvalidation.PredefinedScopeConstraintValidatorManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
@ -30,6 +41,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Service
@ -38,8 +50,6 @@ public class AsyncModelAccessManualService {
@Resource
private ModelService modelService;
@Resource
private ConversationService conversationService;
@Resource
private DatasetQuestionService datasetQuestionService;
@Resource
private ManualModelAnswerMapper manualModelAnswerMapper;
@ -51,10 +61,119 @@ public class AsyncModelAccessManualService {
private ModelAssessTaskDimensionBackupMapper modelAssessTaskDimensionBackupMapper;
@Resource
private ManualModelAnswerBackupMapper manualModelAnswerBackupMapper;
@Resource
private ModelAssessTaskAutoMapper modelAssessTaskAutoMapper;
@Resource
private TrainHttpService trainHttpService;
@Resource
private ModelServiceMapper modelServiceMapper;
@Resource
private BaseModelMapper baseModelMapper;
@Resource
private ModelAssessTaskAutoInfoMapper modelAssessTaskAutoInfoMapper;
@Async
public void auto(ModelAssessTaskAutoDO modelAssessTaskAuto) {
try {
List<DatasetQuestionRespVO> datasetQuestionList = datasetQuestionService.getDatasetQuestionList(modelAssessTaskAuto.getDataset());
for (DatasetQuestionRespVO datasetQuestionRespVO : datasetQuestionList) {
String question = datasetQuestionRespVO.getQuestion();
DatasetAnswerRespVO datasetAnswerRespVO = datasetQuestionRespVO.getDatasetAnswerRespVO().get(0);
String datasetPrompt = datasetAnswerRespVO.getAnswer();
ModelCompletionsReqVO modelCompletionsReqVO = new ModelCompletionsReqVO();
ModelCompletionsReqVO.ModelCompletionsMessage message = new ModelCompletionsReqVO.ModelCompletionsMessage();
if (question != null){
message.setContent(question);
}
message.setRole("user");
List<ModelCompletionsReqVO.ModelCompletionsMessage> messages = new ArrayList<>();
messages.add(message);
modelCompletionsReqVO.setMessages(messages);
ModelCompletionsRespVO modelCompletionsRespVO = modelService.modelCompletions(modelCompletionsReqVO);
String prompt = modelCompletionsRespVO.getAnswer();
String res = trainHttpService.autoEvaluation(prompt, datasetPrompt);
ModelAssessTaskAutoInfoDO infoDO = new ModelAssessTaskAutoInfoDO();
infoDO.setPrompt(prompt);
infoDO.setDataSetPrompt(datasetPrompt);
infoDO.setQuestion(question);
infoDO.setTaskId(modelAssessTaskAuto.getId());
if(modelAssessTaskAuto.getModelType() == 0){
ModelServiceDO modelServiceDO = modelServiceMapper.selectById(modelAssessTaskAuto.getModelService());
if (modelServiceDO != null){
infoDO.setModelName(modelServiceDO.getServiceName());
}
}else{
BaseModelDO baseModelDO = baseModelMapper.selectById(modelAssessTaskAuto.getModelService());
if (baseModelDO != null){
infoDO.setModelName(baseModelDO.getModelName());
}
}
String[] split = res.split("\n");
List<String> list = new ArrayList<>(Arrays.asList(split));
if(list.size() < 3){
if(res.contains("good")){
infoDO.setEmotionTendency(2);
infoDO.setFactualError(2);
infoDO.setSemanticsCoherence(2);
}else if(res.contains("same")){
infoDO.setEmotionTendency(1);
infoDO.setFactualError(1);
infoDO.setSemanticsCoherence(1);
}else if(res.contains("bad")){
infoDO.setEmotionTendency(0);
infoDO.setFactualError(0);
infoDO.setSemanticsCoherence(0);
}
}else{
for (String s : list) {
if(s.contains("事实性错误")){
if(s.contains("good")){
infoDO.setFactualError(2);
}else if (s.contains("same")){
infoDO.setFactualError(1);
}else if (s.contains("bad")){
infoDO.setFactualError(0);
};
}else if(s.contains("事实性错误")){
if(s.contains("good")){
infoDO.setSemanticsCoherence(2);
}else if (s.contains("same")){
infoDO.setSemanticsCoherence(1);
}else if (s.contains("bad")){
infoDO.setSemanticsCoherence(0);
};
}else if(s.contains("情感倾向性")){
if(s.contains("good")){
infoDO.setEmotionTendency(2);
}else if (s.contains("same")){
infoDO.setEmotionTendency(1);
}else if (s.contains("bad")){
infoDO.setEmotionTendency(0);
};
}
}
}
//评估结果: \n\n- 事实性错误: bad\n- 情感倾向性: same\n- 语义连贯性: same
//评估结果: \n- 事实性错误: bad\n- 情感倾向性: same\n- 语义连贯性: good
//事实性错误: bad \n情感倾向性: same \n语义连贯性: bad
//评估结果: good
modelAssessTaskAutoInfoMapper.insert(infoDO);
}
modelAssessTaskAuto.setStatus(4);
modelAssessTaskAutoMapper.updateById(modelAssessTaskAuto);
}catch (Exception e){
modelAssessTaskAuto.setStatus(5);
modelAssessTaskAutoMapper.updateById(modelAssessTaskAuto);
}
}
@Async
public void startModelAccess(ModelAssessTaskManualDO task) {

View File

@ -6,6 +6,7 @@ import cn.iocoder.yudao.framework.common.util.http.HttpUtils;
import cn.iocoder.yudao.module.llm.framework.backend.config.LLMBackendProperties;
import cn.iocoder.yudao.module.llm.service.http.vo.*;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import kong.unirest.HttpResponse;
import kong.unirest.Unirest;
import kong.unirest.UnirestException;
@ -21,8 +22,12 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* 训练相关接口
@ -214,6 +219,27 @@ public class TrainHttpService {
return res;
}
public String autoEvaluation(String prompt,String dataSetPrompt) {
Map<String,String> map = new HashMap<>();
map.put("accept","application/json");
map.put("Content-Type","application/json");
String autoEvaluation = llmBackendProperties.getAutoEvaluation();
JSONObject jsonObject = new JSONObject();
jsonObject.put("prompt",prompt);
jsonObject.put("dataSetPrompt",dataSetPrompt);
List<String> dimension = Stream.of("事实性错误","情感倾向性","语义连贯性").collect(Collectors.toList());
jsonObject.put("dimension",dimension);
String body = JSONObject.toJSONString(jsonObject);
String res = HttpUtils.post(autoEvaluation, map, body);
log.info(" autoEvaluation:{}", res);
JSONObject resJson = JSONObject.parseObject(res);
JSONObject response = resJson.getJSONObject("response");
String content = response.getString("content");
return content;
}
/**
* 模型对话 POST
*/
@ -222,6 +248,7 @@ public class TrainHttpService {
String res = HttpUtils.post(finetuningCreate, headers, body);
return res;
}
public AigcRespVO modelDeploy(Map<String, String> headers,AigcModelDeploySaveReq req,Long jobId){
login(headers);
String modelDeploy = llmBackendProperties.getModelDeploy();
@ -293,4 +320,5 @@ public class TrainHttpService {
}
}
}

View File

@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.llm.service.modelassesstaskauto;
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.llm.controller.admin.dataset.vo.DatasetQuestionRespVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskmanual.vo.ModelAssessTaskManualRespVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.basemodel.BaseModelDO;
import cn.iocoder.yudao.module.llm.dal.dataobject.dataset.DatasetDO;
@ -18,6 +19,8 @@ import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskauto.ModelAssessTask
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskauto.ModelAssessTaskStoplistBackupMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskauto.ModelAssessTaskStoplistMapper;
import cn.iocoder.yudao.module.llm.dal.mysql.modelservice.ModelServiceMapper;
import cn.iocoder.yudao.module.llm.service.async.AsyncModelAccessManualService;
import cn.iocoder.yudao.module.llm.service.dataset.DatasetQuestionService;
import com.alibaba.druid.util.StringUtils;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
@ -59,58 +62,63 @@ public class ModelAssessTaskAutoServiceImpl implements ModelAssessTaskAutoServic
@Resource
private DatasetMapper datasetMapper;
@Resource
private ModelAssessStoplistMapper modelAssessStoplistMapper;
@Resource
private BaseModelMapper baseModelMapper;
@Resource
private ModelAssessTaskAutoBackupMapper modelAssessTaskAutoBackupMapper;
@Resource
private ModelAssessTaskStoplistBackupMapper modelAssessTaskStoplistBackupMapper;
@Resource
private AsyncModelAccessManualService asyncModelAccessManualService;
@Override
public Long createModelAssessTaskAuto(ModelAssessTaskAutoSaveReqVO createReqVO) {
// 插入
ModelAssessTaskAutoDO modelAssessTaskAuto = BeanUtils.toBean(createReqVO, ModelAssessTaskAutoDO.class);
modelAssessTaskAuto.setStatus(3);
LambdaUpdateWrapper<ModelAssessTaskAutoDO> eq = new LambdaUpdateWrapper<ModelAssessTaskAutoDO>().eq(ModelAssessTaskAutoDO::getTaskName, createReqVO.getTaskName());
if (modelAssessTaskAutoMapper.selectCount(eq) > 0){
throw exception(new ErrorCode(11000,"任务名称重复"));
}
modelAssessTaskAutoMapper.insert(modelAssessTaskAuto);
List<ModelAssessTaskStoplistSaveReqVO> stoplists = createReqVO.getStoplists();
if (!CollectionUtils.isEmpty(stoplists)){
stoplists.stream().forEach(stoplist -> {
stoplist.setTaskId(modelAssessTaskAuto.getId());
ModelAssessTaskStoplistDO modelAssessTaskStoplistDO = BeanUtils.toBean(stoplist,ModelAssessTaskStoplistDO.class);
modelingTaskStoplistMapper.insert(modelAssessTaskStoplistDO);
});
}
// List<ModelAssessTaskStoplistSaveReqVO> stoplists = createReqVO.getStoplists();
// if (!CollectionUtils.isEmpty(stoplists)){
// stoplists.stream().forEach(stoplist -> {
// stoplist.setTaskId(modelAssessTaskAuto.getId());
// ModelAssessTaskStoplistDO modelAssessTaskStoplistDO = BeanUtils.toBean(stoplist,ModelAssessTaskStoplistDO.class);
// modelingTaskStoplistMapper.insert(modelAssessTaskStoplistDO);
// });
// }
asyncModelAccessManualService.auto(modelAssessTaskAuto);
// 返回
return modelAssessTaskAuto.getId();
}
@Override
public void updateModelAssessTaskAuto(ModelAssessTaskAutoSaveReqVO updateReqVO) {
// 校验存在
validateModelAssessTaskAutoExists(updateReqVO.getId());
// 更新
ModelAssessTaskAutoDO updateObj = BeanUtils.toBean(updateReqVO, ModelAssessTaskAutoDO.class);
List<ModelAssessTaskStoplistSaveReqVO> stoplists = updateReqVO.getStoplists();
if (!CollectionUtils.isEmpty(stoplists)){
List<Long> stopIds = stoplists.stream().filter(stoplist -> stoplist.getId() != null)
.map(stoplist -> stoplist.getId()).collect(Collectors.toList());
LambdaQueryWrapper<ModelAssessTaskStoplistDO> wrapper = new LambdaQueryWrapper<ModelAssessTaskStoplistDO>();
wrapper.eq(ModelAssessTaskStoplistDO::getTaskId,updateReqVO.getId());
if (!CollectionUtils.isEmpty(stopIds)){
wrapper.notIn(ModelAssessTaskStoplistDO::getId,stopIds);
}
modelingTaskStoplistMapper.delete(wrapper);
stoplists.stream().forEach(stoplist -> {
// stoplist.setTaskId(updateObj.getId());
ModelAssessTaskStoplistDO modelAssessTaskStoplistDO = BeanUtils.toBean(stoplist,ModelAssessTaskStoplistDO.class);
modelingTaskStoplistMapper.insertOrUpdate(modelAssessTaskStoplistDO);
});
}
// List<ModelAssessTaskStoplistSaveReqVO> stoplists = updateReqVO.getStoplists();
// if (!CollectionUtils.isEmpty(stoplists)){
// List<Long> stopIds = stoplists.stream().filter(stoplist -> stoplist.getId() != null)
// .map(stoplist -> stoplist.getId()).collect(Collectors.toList());
// LambdaQueryWrapper<ModelAssessTaskStoplistDO> wrapper = new LambdaQueryWrapper<ModelAssessTaskStoplistDO>();
// wrapper.eq(ModelAssessTaskStoplistDO::getTaskId,updateReqVO.getId());
// if (!CollectionUtils.isEmpty(stopIds)){
// wrapper.notIn(ModelAssessTaskStoplistDO::getId,stopIds);
// }
// modelingTaskStoplistMapper.delete(wrapper);
// stoplists.stream().forEach(stoplist -> {
//// stoplist.setTaskId(updateObj.getId());
// ModelAssessTaskStoplistDO modelAssessTaskStoplistDO = BeanUtils.toBean(stoplist,ModelAssessTaskStoplistDO.class);
// modelingTaskStoplistMapper.insertOrUpdate(modelAssessTaskStoplistDO);
// });
// }
modelAssessTaskAutoMapper.updateById(updateObj);
}
@ -136,32 +144,39 @@ public class ModelAssessTaskAutoServiceImpl implements ModelAssessTaskAutoServic
ModelAssessTaskAutoDO modelAssessTaskAutoDO = modelAssessTaskAutoMapper.selectById(id);
ModelAssessTaskAutoRespVO result = BeanUtils.toBean(modelAssessTaskAutoDO, ModelAssessTaskAutoRespVO.class);
// 标注查询
List<ModelAssessTaskStoplistDO> modelAssessTaskStoplistDOS = modelingTaskStoplistMapper.selectList(new LambdaQueryWrapper<ModelAssessTaskStoplistDO>()
.eq(ModelAssessTaskStoplistDO::getTaskId, id)
.eq(ModelAssessTaskStoplistDO::getDeleted, false));
List<ModelAssessTaskStoplistRespVO> stoplistRespVOS = BeanUtils.toBean(modelAssessTaskStoplistDOS, ModelAssessTaskStoplistRespVO.class);
// 提取id
List<Long> stoplistIds = modelAssessTaskStoplistDOS.stream().map(ModelAssessTaskStoplistDO::getStoplistId).collect(Collectors.toList());
if (!CollectionUtils.isEmpty(stoplistIds)){
// 查询停用词表 将词表word返回
LambdaQueryWrapper<ModelAssessStoplistDO> wrapper = new LambdaQueryWrapper<>();
wrapper.in(ModelAssessStoplistDO::getId,stoplistIds);
List<ModelAssessStoplistDO> modelAssessStoplistDOs = modelAssessStoplistMapper.selectList(wrapper);
Map<Long, ModelAssessStoplistDO> longModelServiceDOMap = cn.iocoder.yudao.framework.common.util.collection.
CollectionUtils.convertMap(modelAssessStoplistDOs, ModelAssessStoplistDO::getId);
stoplistRespVOS.stream().forEach(stoplistRespVO -> {
ModelAssessStoplistDO modelAssessStoplistDO = longModelServiceDOMap.get(stoplistRespVO.getStoplistId());
if(modelAssessStoplistDO != null){
stoplistRespVO.setStoplistName(modelAssessStoplistDO.getWord());
}
});
}
result.setStoplists(stoplistRespVOS);
// // 标注查询
// List<ModelAssessTaskStoplistDO> modelAssessTaskStoplistDOS = modelingTaskStoplistMapper.selectList(new LambdaQueryWrapper<ModelAssessTaskStoplistDO>()
// .eq(ModelAssessTaskStoplistDO::getTaskId, id)
// .eq(ModelAssessTaskStoplistDO::getDeleted, false));
// List<ModelAssessTaskStoplistRespVO> stoplistRespVOS = BeanUtils.toBean(modelAssessTaskStoplistDOS, ModelAssessTaskStoplistRespVO.class);
// // 提取id
// List<Long> stoplistIds = modelAssessTaskStoplistDOS.stream().map(ModelAssessTaskStoplistDO::getStoplistId).collect(Collectors.toList());
// if (!CollectionUtils.isEmpty(stoplistIds)){
// // 查询停用词表 将词表word返回
// LambdaQueryWrapper<ModelAssessStoplistDO> wrapper = new LambdaQueryWrapper<>();
// wrapper.in(ModelAssessStoplistDO::getId,stoplistIds);
// List<ModelAssessStoplistDO> modelAssessStoplistDOs = modelAssessStoplistMapper.selectList(wrapper);
// Map<Long, ModelAssessStoplistDO> longModelServiceDOMap = cn.iocoder.yudao.framework.common.util.collection.
// CollectionUtils.convertMap(modelAssessStoplistDOs, ModelAssessStoplistDO::getId);
// stoplistRespVOS.stream().forEach(stoplistRespVO -> {
// ModelAssessStoplistDO modelAssessStoplistDO = longModelServiceDOMap.get(stoplistRespVO.getStoplistId());
// if(modelAssessStoplistDO != null){
// stoplistRespVO.setStoplistName(modelAssessStoplistDO.getWord());
// }
// });
// }
// result.setStoplists(stoplistRespVOS);
ModelServiceDO modelServiceDO = modelServiceMapper.selectById(modelAssessTaskAutoDO.getModelService());
if (modelServiceDO != null){
result.setModelServiceName(modelServiceDO.getServiceName());
if(modelAssessTaskAutoDO.getModelType() == 0){
ModelServiceDO modelServiceDO = modelServiceMapper.selectById(modelAssessTaskAutoDO.getModelService());
if (modelServiceDO != null){
result.setModelServiceName(modelServiceDO.getServiceName());
}
}else{
BaseModelDO baseModelDO = baseModelMapper.selectById(modelAssessTaskAutoDO.getModelService());
if (baseModelDO != null){
result.setModelServiceName(baseModelDO.getModelName());
}
}
DatasetDO datasetDO = datasetMapper.selectById(modelAssessTaskAutoDO.getDataset());
if (datasetDO != null){

View File

@ -0,0 +1,59 @@
package cn.iocoder.yudao.module.llm.service.modelassesstaskautoinfo;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo.ModelAssessTaskAutoInfoDO;
import javax.validation.Valid;
import java.util.List;
/**
* 模型评估任务 自动评估-详细内容 Service 接口
*
* @author 华大大模型
*/
public interface ModelAssessTaskAutoInfoService {
/**
* 创建模型评估任务 自动评估-详细内容
*
* @param createReqVO 创建信息
* @return 编号
*/
Long createModelAssessTaskAutoInfo(@Valid ModelAssessTaskAutoInfoSaveReqVO createReqVO);
/**
* 更新模型评估任务 自动评估-详细内容
*
* @param updateReqVO 更新信息
*/
void updateModelAssessTaskAutoInfo(@Valid ModelAssessTaskAutoInfoSaveReqVO updateReqVO);
/**
* 删除模型评估任务 自动评估-详细内容
*
* @param id 编号
*/
void deleteModelAssessTaskAutoInfo(Long id);
/**
* 获得模型评估任务 自动评估-详细内容
*
* @param id 编号
* @return 模型评估任务 自动评估-详细内容
*/
ModelAssessTaskAutoInfoDO getModelAssessTaskAutoInfo(Long id);
/**
* 获得模型评估任务 自动评估-详细内容分页
*
* @param pageReqVO 分页查询
* @return 模型评估任务 自动评估-详细内容分页
*/
PageResult<ModelAssessTaskAutoInfoDO> getModelAssessTaskAutoInfoPage(ModelAssessTaskAutoInfoPageReqVO pageReqVO);
List<ModelAssessTaskAutoInfoDO> getListByTaskId(ModelAssessTaskAutoInfoPageReqVO pageReqVO);
}

View File

@ -0,0 +1,71 @@
package cn.iocoder.yudao.module.llm.service.modelassesstaskautoinfo;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoPageReqVO;
import cn.iocoder.yudao.module.llm.controller.admin.modelassesstaskauto.vo.ModelAssessTaskAutoInfoSaveReqVO;
import cn.iocoder.yudao.module.llm.dal.dataobject.modelassesstaskautoinfo.ModelAssessTaskAutoInfoDO;
import cn.iocoder.yudao.module.llm.dal.mysql.modelassesstaskautoinfo.ModelAssessTaskAutoInfoMapper;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
import javax.annotation.Resource;
import java.util.List;
/**
* 模型评估任务 自动评估-详细内容 Service 实现类
*
* @author 华大大模型
*/
@Service
@Validated
public class ModelAssessTaskAutoInfoServiceImpl implements ModelAssessTaskAutoInfoService {
@Resource
private ModelAssessTaskAutoInfoMapper modelAssessTaskAutoInfoMapper;
@Override
public Long createModelAssessTaskAutoInfo(ModelAssessTaskAutoInfoSaveReqVO createReqVO) {
// 插入
ModelAssessTaskAutoInfoDO modelAssessTaskAutoInfo = BeanUtils.toBean(createReqVO, ModelAssessTaskAutoInfoDO.class);
modelAssessTaskAutoInfoMapper.insert(modelAssessTaskAutoInfo);
// 返回
return modelAssessTaskAutoInfo.getId();
}
@Override
public void updateModelAssessTaskAutoInfo(ModelAssessTaskAutoInfoSaveReqVO updateReqVO) {
// 更新
ModelAssessTaskAutoInfoDO updateObj = BeanUtils.toBean(updateReqVO, ModelAssessTaskAutoInfoDO.class);
modelAssessTaskAutoInfoMapper.updateById(updateObj);
}
@Override
public void deleteModelAssessTaskAutoInfo(Long id) {
// 删除
modelAssessTaskAutoInfoMapper.deleteById(id);
}
@Override
public ModelAssessTaskAutoInfoDO getModelAssessTaskAutoInfo(Long id) {
return modelAssessTaskAutoInfoMapper.selectById(id);
}
@Override
public PageResult<ModelAssessTaskAutoInfoDO> getModelAssessTaskAutoInfoPage(ModelAssessTaskAutoInfoPageReqVO pageReqVO) {
return modelAssessTaskAutoInfoMapper.selectPage(pageReqVO);
}
@Override
public List<ModelAssessTaskAutoInfoDO> getListByTaskId(ModelAssessTaskAutoInfoPageReqVO pageReqVO) {
LambdaQueryWrapper<ModelAssessTaskAutoInfoDO> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(ModelAssessTaskAutoInfoDO::getTaskId,pageReqVO.getTaskId());
return modelAssessTaskAutoInfoMapper.selectList(queryWrapper);
}
}

View File

@ -8,7 +8,7 @@ import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.List;
/**
/**z
* @Description : 海明距离算法
*/
@Slf4j

View File

@ -264,9 +264,9 @@ llm:
# aigc模型推理
aigc_model_completions: http://36.133.1.230:8080/api/channels/chat/completions
application_api: http://localhost:48100/admin-app/llm/application/api/apiKey/chat
application_api: http://xhllm.xinnuojinzhi.com/admin-app/llm/application/api/apiKey/chat
model_service_api: http://localhost:48100/admin-app/llm/model-service/api/apiKey/chat
model_service_api: http://xhllm.xinnuojinzhi.com/admin-app/llm/model-service/api/apiKey/chat
model_file_list: http://36.133.1.230:5123/models/?path=
@ -274,6 +274,8 @@ llm:
# 提示词优化
optimize_prompt: http://36.133.1.230:5123/optimize-prompt
auto_evaluation: http://36.133.1.230:5123/llm-eval
--- #################### iot相关配置 TODO 芋艿:再瞅瞅 ####################
iot:

View File

@ -316,6 +316,8 @@ llm:
# 提示词优化
optimize_prompt: http://36.133.1.230:5123/optimize-prompt
auto_evaluation: http://36.133.1.230:5123/llm-eval
--- #################### iot相关配置 TODO 芋艿:再瞅瞅 ####################
iot:
emq: