[update] 模型调优删除时校验模型服务管理有没有真正使用
This commit is contained in:
parent
77890fe610
commit
56e4ba790f
@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.llm.controller.admin.finetuningtask.vo.FineTuning
|
||||
import cn.iocoder.yudao.module.llm.dal.dataobject.application.ApplicationDO;
|
||||
import cn.iocoder.yudao.module.llm.service.conversation.ConversationService;
|
||||
import cn.iocoder.yudao.module.llm.service.finetuningtask.FineTuningTaskService;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import javax.annotation.Resource;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
@ -43,10 +44,14 @@ import cn.iocoder.yudao.module.llm.service.modelservice.ModelServiceService;
|
||||
@Validated
|
||||
public class ModelServiceController {
|
||||
|
||||
@Lazy
|
||||
@Resource
|
||||
private ModelServiceService modelServiceService;
|
||||
|
||||
@Lazy
|
||||
@Resource
|
||||
private FineTuningTaskService fineTuningTaskService;
|
||||
|
||||
@Resource
|
||||
private ConversationService conversationService;
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package cn.iocoder.yudao.module.llm.service.finetuningtask;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.exception.ErrorCode;
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
|
||||
import cn.iocoder.yudao.module.llm.controller.admin.finetuningtask.vo.FineTuningTaskPageReqVO;
|
||||
@ -15,7 +16,7 @@ import cn.iocoder.yudao.module.llm.enums.FinetuningTaskStatusEnum;
|
||||
import cn.iocoder.yudao.module.llm.service.async.AsyncFineTuningTaskService;
|
||||
import cn.iocoder.yudao.module.llm.service.http.TrainHttpService;
|
||||
import cn.iocoder.yudao.module.llm.service.http.vo.AigcFineTuningDetailRespVO;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import cn.iocoder.yudao.module.llm.service.modelservice.ModelServiceService;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
|
||||
@ -34,7 +35,8 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
|
||||
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.*;
|
||||
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.FINE_TUNING_TASK_NAME_NOT_EXISTS;
|
||||
import static cn.iocoder.yudao.module.llm.enums.ErrorCodeConstants.FINE_TUNING_TASK_NOT_EXISTS;
|
||||
|
||||
|
||||
/**
|
||||
@ -58,8 +60,11 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
@Resource
|
||||
TrainHttpService trainHttpService;
|
||||
|
||||
@Resource
|
||||
private ModelServiceService modelServiceService;
|
||||
|
||||
@Override
|
||||
public Long createFineTuningTask(FineTuningTaskSaveReqVO createReqVO) {
|
||||
public Long createFineTuningTask (FineTuningTaskSaveReqVO createReqVO) {
|
||||
// 插入
|
||||
validateFineTuningNameTaskExists(createReqVO);
|
||||
FineTuningTaskDO fineTuningTask = BeanUtils.toBean(createReqVO, FineTuningTaskDO.class);
|
||||
@ -72,7 +77,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateFineTuningTask(FineTuningTaskSaveReqVO updateReqVO) {
|
||||
public void updateFineTuningTask (FineTuningTaskSaveReqVO updateReqVO) {
|
||||
// 校验存在
|
||||
validateFineTuningTaskExists(updateReqVO.getId());
|
||||
validateFineTuningNameTaskExists(updateReqVO);
|
||||
@ -82,77 +87,111 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFineTuningTask(Long id) {
|
||||
public void deleteFineTuningTask (Long id) {
|
||||
// 校验存在
|
||||
validateFineTuningTaskExists(id);
|
||||
// 校验使用
|
||||
validateFineTuningTaskUse(id);
|
||||
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(id);
|
||||
// 删除
|
||||
LambdaUpdateWrapper<FineTuningTaskDO> wrapper = new LambdaUpdateWrapper<>();
|
||||
wrapper.eq(FineTuningTaskDO::getId, id)
|
||||
.set(FineTuningTaskDO::getDeleted,true);
|
||||
fineTuningTaskMapper.update(null,wrapper);
|
||||
.set(FineTuningTaskDO::getDeleted, true);
|
||||
fineTuningTaskMapper.update(null, wrapper);
|
||||
asyncFineTuningTaskService.stopFineTuningTask(fineTuningTaskDO);
|
||||
// fineTuningTaskMapper.deleteById(id);
|
||||
// fineTuningTaskMapper.deleteById(id);
|
||||
}
|
||||
|
||||
private FineTuningTaskDO validateFineTuningTaskExists(Long id) {
|
||||
/**
|
||||
* 校验使用
|
||||
* <p>
|
||||
* 删除调优任务时，需要判断是否正在使用，如果正在使用，则不允许删除
|
||||
* </p>
|
||||
*
|
||||
* @param id 任务id
|
||||
*/
|
||||
private void validateFineTuningTaskUse (Long id) {
|
||||
FineTuningTaskDO fineTuningTask = fineTuningTaskMapper.selectById(id);
|
||||
String taskName = fineTuningTask.getTaskName();
|
||||
|
||||
// 校验 微调任务 是否在 模型服务管理 有使用
|
||||
validateTaskUsesInModelService(id, taskName);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验微调任务是否在模型服务管理中使用
|
||||
*
|
||||
* @param id 任务id
|
||||
* @param taskName 任务名称
|
||||
*/
|
||||
private void validateTaskUsesInModelService (Long id, String taskName) {
|
||||
Map<Long, String> modelService = modelServiceService.getModelServiceByTaskId(id);
|
||||
if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isNotEmpty(modelService)) {
|
||||
String msg = String.format("微调任务【%s】在模型服务管理 %s 有使用，请先取消使用", taskName, modelService.values());
|
||||
throw exception(new ErrorCode(10005_001, msg));
|
||||
}
|
||||
}
|
||||
|
||||
private FineTuningTaskDO validateFineTuningTaskExists (Long id) {
|
||||
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(id);
|
||||
if (fineTuningTaskDO == null) {
|
||||
throw exception(FINE_TUNING_TASK_NOT_EXISTS);
|
||||
}
|
||||
return fineTuningTaskDO;
|
||||
}
|
||||
private void validateFineTuningNameTaskExists(FineTuningTaskSaveReqVO reqVO) {
|
||||
|
||||
private void validateFineTuningNameTaskExists (FineTuningTaskSaveReqVO reqVO) {
|
||||
LambdaQueryWrapper<FineTuningTaskDO> wrapper = new LambdaQueryWrapper<FineTuningTaskDO>()
|
||||
.eq(FineTuningTaskDO::getTaskName, reqVO.getTaskName());
|
||||
|
||||
if (reqVO.getId() != null){
|
||||
if (reqVO.getId() != null) {
|
||||
wrapper.ne(FineTuningTaskDO::getId, reqVO.getId());
|
||||
}
|
||||
List<FineTuningTaskDO> dineTuningTaskDos = fineTuningTaskMapper.selectList(wrapper);
|
||||
if (CollectionUtils.isNotEmpty(dineTuningTaskDos)){
|
||||
if (CollectionUtils.isNotEmpty(dineTuningTaskDos)) {
|
||||
throw exception(FINE_TUNING_TASK_NAME_NOT_EXISTS);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public FineTuningTaskRespVO getFineTuningTask(Long id) {
|
||||
public FineTuningTaskRespVO getFineTuningTask (Long id) {
|
||||
validateFineTuningTaskExists(id);
|
||||
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(id);
|
||||
FineTuningTaskRespVO result = BeanUtils.toBean(fineTuningTaskDO, FineTuningTaskRespVO.class);
|
||||
Long dataset = fineTuningTaskDO.getDataset();
|
||||
if (dataset != null) {
|
||||
DatasetDO datasetDO = datasetMapper.selectById(dataset);
|
||||
if (datasetDO != null){
|
||||
if (datasetDO != null) {
|
||||
result.setDatasetName(datasetDO.getDatasetName());
|
||||
}
|
||||
}
|
||||
Long baseModelId = fineTuningTaskDO.getBaseModelId();
|
||||
if (baseModelId != null) {
|
||||
BaseModelDO baseModelDO = baseModelMapper.selectById(baseModelId);
|
||||
if (baseModelDO != null){
|
||||
if (baseModelDO != null) {
|
||||
result.setBaseModelName(baseModelDO.getModelName());
|
||||
}
|
||||
}
|
||||
if (StringUtil.isNotEmpty(fineTuningTaskDO.getJobId())){
|
||||
if (StringUtil.isNotEmpty(fineTuningTaskDO.getJobId())) {
|
||||
AigcFineTuningDetailRespVO aigcFineTuningDetailRespVO = trainHttpService.finetuningDetail(new HashMap<>(), fineTuningTaskDO.getJobId());
|
||||
result.setDetailRespVO(aigcFineTuningDetailRespVO);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<FineTuningTaskRespVO> getFineTuningTaskPage1(FineTuningTaskPageReqVO pageReqVO) {
|
||||
public PageResult<FineTuningTaskRespVO> getFineTuningTaskPage1 (FineTuningTaskPageReqVO pageReqVO) {
|
||||
PageResult<FineTuningTaskDO> pageResult = fineTuningTaskMapper.selectPage(pageReqVO);
|
||||
PageResult<FineTuningTaskRespVO> result = BeanUtils.toBean(pageResult, FineTuningTaskRespVO.class);
|
||||
if (CollectionUtils.isNotEmpty(pageResult.getList())){
|
||||
if (CollectionUtils.isNotEmpty(pageResult.getList())) {
|
||||
List<Long> modelIds = pageResult.getList().stream().map(FineTuningTaskDO::getBaseModelId).collect(Collectors.toList());
|
||||
List<BaseModelDO> baseModelDOS = baseModelMapper.selectList(new LambdaQueryWrapper<BaseModelDO>()
|
||||
.in(BaseModelDO::getId, modelIds));
|
||||
Map<Long, BaseModelDO> longModelServiceDOMap = cn.iocoder.yudao.framework.common.util.collection.
|
||||
CollectionUtils.convertMap(baseModelDOS, BaseModelDO::getId);
|
||||
result.getList().forEach(item->{
|
||||
BaseModelDO baseModelDO = longModelServiceDOMap.get(item.getBaseModelId());
|
||||
if(baseModelDO != null){
|
||||
result.getList().forEach(item -> {
|
||||
BaseModelDO baseModelDO = longModelServiceDOMap.get(item.getBaseModelId());
|
||||
if (baseModelDO != null) {
|
||||
item.setBaseModelName(baseModelDO.getModelName());
|
||||
}
|
||||
});
|
||||
@ -160,9 +199,9 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
List<DatasetDO> datasetDOS = datasetMapper.selectList(new LambdaQueryWrapper<DatasetDO>().in(DatasetDO::getId, dataSetlIds));
|
||||
Map<Long, DatasetDO> datasetDOMap = cn.iocoder.yudao.framework.common.util.collection.
|
||||
CollectionUtils.convertMap(datasetDOS, DatasetDO::getId);
|
||||
result.getList().forEach(item->{
|
||||
result.getList().forEach(item -> {
|
||||
DatasetDO datasetDO = datasetDOMap.get(item.getDataset());
|
||||
if(datasetDO != null){
|
||||
if (datasetDO != null) {
|
||||
item.setDatasetName(datasetDO.getDatasetName());
|
||||
}
|
||||
});
|
||||
@ -173,7 +212,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
|
||||
|
||||
@Override
|
||||
public void startFineTuningTask(Long id) {
|
||||
public void startFineTuningTask (Long id) {
|
||||
FineTuningTaskDO fineTuningTaskDO = validateFineTuningTaskExists(id);
|
||||
fineTuningTaskMapper.stopStartTask(id, FinetuningTaskStatusEnum.WAITING.getStatus());
|
||||
//调用模型服务，开启调优任务
|
||||
@ -181,7 +220,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<FineTuningTaskRespVO> selectAll() {
|
||||
public List<FineTuningTaskRespVO> selectAll () {
|
||||
List<FineTuningTaskDO> fineTuningTaskDOS = fineTuningTaskMapper.selectList();
|
||||
return BeanUtils.toBean(fineTuningTaskDOS, FineTuningTaskRespVO.class);
|
||||
}
|
||||
@ -195,7 +234,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
public List<FineTuningTaskRespVO> selectEnable () {
|
||||
List<FineTuningTaskDO> fineTuningTaskDOS = fineTuningTaskMapper.selectList();
|
||||
fineTuningTaskDOS = fineTuningTaskDOS.stream()
|
||||
.filter(item->item.getStatus().equals(FinetuningTaskStatusEnum.FINISHED.getStatus()))
|
||||
.filter(item -> item.getStatus().equals(FinetuningTaskStatusEnum.FINISHED.getStatus()))
|
||||
.collect(Collectors.toList());
|
||||
return BeanUtils.toBean(fineTuningTaskDOS, FineTuningTaskRespVO.class);
|
||||
}
|
||||
@ -209,15 +248,15 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
@Override
|
||||
public Map<Long, String> getModelTuningByModelId (Long baseModelId) {
|
||||
|
||||
if (baseModelId == null){
|
||||
if (baseModelId == null) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
List<FineTuningTaskDO> fineTuningTaskDOS = fineTuningTaskMapper.selectList(new LambdaQueryWrapper<FineTuningTaskDO>()
|
||||
.eq(FineTuningTaskDO::getBaseModelId, baseModelId)
|
||||
.in(FineTuningTaskDO::getStatus, FinetuningTaskStatusEnum.WAITING.getStatus(),FinetuningTaskStatusEnum.TRAINING.getStatus()));
|
||||
.in(FineTuningTaskDO::getStatus, FinetuningTaskStatusEnum.WAITING.getStatus(), FinetuningTaskStatusEnum.TRAINING.getStatus()));
|
||||
|
||||
if (CollectionUtils.isNotEmpty(fineTuningTaskDOS)){
|
||||
if (CollectionUtils.isNotEmpty(fineTuningTaskDOS)) {
|
||||
return fineTuningTaskDOS.stream().collect(Collectors.toMap(FineTuningTaskDO::getId, FineTuningTaskDO::getTaskName));
|
||||
}
|
||||
|
||||
@ -232,15 +271,15 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
*/
|
||||
@Override
|
||||
public Map<Long, String> getModelTuningByDatasetId (Long datasetId) {
|
||||
if (datasetId == null){
|
||||
if (datasetId == null) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
List<FineTuningTaskDO> fineTuningTaskDOS = fineTuningTaskMapper.selectList(new LambdaQueryWrapper<FineTuningTaskDO>()
|
||||
.eq(FineTuningTaskDO::getDataset, datasetId)
|
||||
.in(FineTuningTaskDO::getStatus, FinetuningTaskStatusEnum.WAITING.getStatus(),FinetuningTaskStatusEnum.TRAINING.getStatus()));
|
||||
.in(FineTuningTaskDO::getStatus, FinetuningTaskStatusEnum.WAITING.getStatus(), FinetuningTaskStatusEnum.TRAINING.getStatus()));
|
||||
|
||||
if (CollectionUtils.isNotEmpty(fineTuningTaskDOS)){
|
||||
if (CollectionUtils.isNotEmpty(fineTuningTaskDOS)) {
|
||||
return fineTuningTaskDOS.stream().collect(Collectors.toMap(FineTuningTaskDO::getId, FineTuningTaskDO::getTaskName));
|
||||
}
|
||||
|
||||
@ -248,18 +287,18 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFineTuningLog(Long id) {
|
||||
public String getFineTuningLog (Long id) {
|
||||
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(id);
|
||||
if (fineTuningTaskDO != null && StringUtils.isNotBlank(fineTuningTaskDO.getTrainLog())){
|
||||
Map<String,String> map = new HashMap<>();
|
||||
map.put("log_path",fineTuningTaskDO.getTrainDuration());
|
||||
if (fineTuningTaskDO != null && StringUtils.isNotBlank(fineTuningTaskDO.getTrainLog())) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put("log_path", fineTuningTaskDO.getTrainDuration());
|
||||
return trainHttpService.getTrainLog(map);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopFineTuningTask(Long id) {
|
||||
public void stopFineTuningTask (Long id) {
|
||||
FineTuningTaskDO fineTuningTaskDO = validateFineTuningTaskExists(id);
|
||||
fineTuningTaskMapper.stopStartTask(id, FinetuningTaskStatusEnum.WAITING.getStatus());
|
||||
//todo 调用模型服务，停止调优任务
|
||||
@ -269,7 +308,7 @@ public class FineTuningTaskServiceImpl implements FineTuningTaskService {
|
||||
|
||||
|
||||
@Override
|
||||
public PageResult<FineTuningTaskDO> getFineTuningTaskPage(FineTuningTaskPageReqVO pageReqVO) {
|
||||
public PageResult<FineTuningTaskDO> getFineTuningTaskPage (FineTuningTaskPageReqVO pageReqVO) {
|
||||
return fineTuningTaskMapper.selectPage(pageReqVO);
|
||||
}
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ import cn.iocoder.yudao.module.llm.dal.dataobject.modelservice.ModelServiceDO;
|
||||
|
||||
import javax.validation.Valid;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 模型服务 Service 接口
|
||||
@ -77,4 +78,13 @@ public interface ModelServiceService {
|
||||
* @return 模型服务管理数量
|
||||
*/
|
||||
Long getCount ();
|
||||
|
||||
|
||||
/**
|
||||
* 根据 微调任务 id 获取 模型服务
|
||||
*
|
||||
* @param taskId 知识库 id
|
||||
* @return 模型服务的 Map 集合
|
||||
*/
|
||||
Map<Long, String> getModelServiceByTaskId (Long taskId);
|
||||
}
|
||||
|
||||
@ -26,10 +26,7 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
|
||||
@ -89,9 +86,12 @@ public class ModelServiceServiceImpl implements ModelServiceService {
|
||||
|
||||
FineTuningTaskDO fineTuningTaskDO = fineTuningTaskMapper.selectById(createReqVO.getFineTuningTask());
|
||||
|
||||
//aigc模型列表中的id 以及 对话用的modelName
|
||||
modelService.setJobId(fineTuningTaskDO.getJobModelListId());
|
||||
modelService.setBaseModelName(fineTuningTaskDO.getJobModelName());
|
||||
if (fineTuningTaskDO != null){
|
||||
//aigc模型列表中的id 以及 对话用的modelName
|
||||
modelService.setJobId(fineTuningTaskDO.getJobModelListId());
|
||||
modelService.setBaseModelName(fineTuningTaskDO.getJobModelName());
|
||||
|
||||
}
|
||||
|
||||
// 插入
|
||||
String apikey = getApikey();
|
||||
@ -326,4 +326,22 @@ public class ModelServiceServiceImpl implements ModelServiceService {
|
||||
return modelServiceMapper.selectCount(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 微调任务 id 获取 模型服务
|
||||
*
|
||||
* @param taskId 知识库 id
|
||||
* @return 模型服务的 Map 集合
|
||||
*/
|
||||
@Override
|
||||
public Map<Long, String> getModelServiceByTaskId (Long taskId) {
|
||||
if (taskId != null) {
|
||||
List<ModelServiceDO> modelServiceDOS = modelServiceMapper.selectList(new LambdaQueryWrapper<ModelServiceDO>()
|
||||
.eq(ModelServiceDO::getFineTuningTask, taskId));
|
||||
if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isNotEmpty(modelServiceDOS)) {
|
||||
return modelServiceDOS.stream().collect(Collectors.toMap(ModelServiceDO::getId, ModelServiceDO::getServiceName));
|
||||
}
|
||||
}
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user