Java tutorial
/* * Copyright (C) 2017 Baifendian Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.baifendian.swordfish.webserver.service; import static com.baifendian.swordfish.webserver.utils.ParamVerify.flowNodeParamCheck; import static com.baifendian.swordfish.webserver.utils.ParamVerify.verifyDesc; import static com.baifendian.swordfish.webserver.utils.ParamVerify.verifyExtras; import static com.baifendian.swordfish.webserver.utils.ParamVerify.verifyProxyUser; import static com.baifendian.swordfish.webserver.utils.ParamVerify.verifyWorkflowName; import com.baifendian.swordfish.common.config.BaseConfig; import com.baifendian.swordfish.common.hadoop.HdfsClient; import com.baifendian.swordfish.common.utils.graph.Graph; import com.baifendian.swordfish.dao.FlowDao; import com.baifendian.swordfish.dao.mapper.ProjectFlowMapper; import com.baifendian.swordfish.dao.mapper.ProjectMapper; import com.baifendian.swordfish.dao.model.FlowNode; import com.baifendian.swordfish.dao.model.Project; import com.baifendian.swordfish.dao.model.ProjectFlow; import com.baifendian.swordfish.dao.model.User; import com.baifendian.swordfish.dao.utils.json.JsonUtil; import com.baifendian.swordfish.webserver.dto.WorkflowData; import com.baifendian.swordfish.webserver.dto.WorkflowNodeDto; import com.baifendian.swordfish.webserver.exception.BadRequestException; import com.baifendian.swordfish.webserver.exception.NotFoundException; import com.baifendian.swordfish.webserver.exception.ParameterException; import com.baifendian.swordfish.webserver.exception.PermissionException; import com.baifendian.swordfish.webserver.exception.ServerErrorException; import com.baifendian.swordfish.webserver.service.storage.FileSystemStorageService; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.FileInputStream; import java.io.IOException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.UUID; import net.lingala.zip4j.core.ZipFile; import net.lingala.zip4j.exception.ZipException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @Service public class WorkflowService { private static Logger logger = LoggerFactory.getLogger(WorkflowService.class.getName()); private static final String WorkflowJson = "workflow.json"; @Autowired private ProjectFlowMapper projectFlowMapper; @Autowired private ProjectMapper projectMapper; @Autowired private ProjectService projectService; @Autowired private FileSystemStorageService fileSystemStorageService; @Autowired private ExecService execService; @Autowired private FlowDao flowDao; /** * ?, ? \"w\" ?? * * @param operator ? * @param projectName ??? * @param name ??? * @param desc ??? * @param proxyUser ???? * @param queue ??? * @param data ?json * @param file ? * @param extras ??? * @param flag ?, 0 , 1 ?, ???? * @return ?? */ public ProjectFlow createWorkflow(User operator, String projectName, String name, String desc, String proxyUser, String queue, String data, MultipartFile file, String extras, Integer flag) { // ?? verifyWorkflowName(name); verifyDesc(desc); verifyExtras(extras); Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ??? if (!projectService.hasWritePerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" write permission", operator.getName(), project.getName()); } // proxyUser ??? verifyProxyUser(operator.getProxyUserList(), proxyUser); // ? json ??? WorkflowData workflowData = workflowDataDes(data, file, name, project); if (workflowData == null) { logger.error("Project flow data or file not valid"); throw new ParameterException("Data \"{0}\" not valid", data); } // ?? List<WorkflowNodeDto> flowNodes = workflowData.getNodes(); if (CollectionUtils.isEmpty(flowNodes)) { logger.error("flow node information is empty"); throw new ParameterException("Data \"{0}\" is null", data); } // ? if (graphHasCycle(flowNodes)) { logger.error("Proejct flow DAG has cycle"); throw new ParameterException("Flow node has cycle"); } // ? json ? for (WorkflowNodeDto flowNode : flowNodes) { if (!flowNodeParamCheck(flowNode.getParameter(), flowNode.getType())) { logger.error("Flow node {} parameter invalid", flowNode.getName()); throw new ParameterException("Flow node \"{0}\" parameter invalid ", flowNode.getName()); } // ?? verifyExtras(flowNode.getExtras()); } ProjectFlow projectFlow = new ProjectFlow(); Date now = new Date(); // ?? try { List<FlowNode> flowNodeList = new ArrayList<>(); for (WorkflowNodeDto flowNode : flowNodes) { flowNodeList.add(flowNode.convertFlowNode()); } projectFlow.setName(name); projectFlow.setProjectId(project.getId()); projectFlow.setProjectName(projectName); projectFlow.setDesc(desc); projectFlow.setCreateTime(now); projectFlow.setModifyTime(now); projectFlow.setProxyUser(proxyUser); projectFlow.setQueue(queue); projectFlow.setOwnerId(operator.getId()); projectFlow.setOwner(operator.getName()); projectFlow.setExtras(extras); projectFlow.setFlowsNodes(flowNodeList); projectFlow.setUserDefinedParamList(workflowData.getUserDefParams()); projectFlow.setFlag(flag); } catch (Exception e) { logger.error("Project flow set value error", e); throw new BadRequestException("Project flow set value error", e); } try { flowDao.createProjectFlow(projectFlow); } catch (DuplicateKeyException e) { logger.error("Workflow has exist, can't create again.", e); throw new ServerErrorException("Workflow has exist, can't create again."); } catch (Exception e) { logger.error("Workflow create has error", e); throw new ServerErrorException("Workflow create has error", e); } return projectFlow; } /** * ???? \"w\" ?? * <p> * ??,null * </p> * * @param operator * @param projectName * @param name * @param desc * @param proxyUser * @param queue * @param data * @param file * @return */ public ProjectFlow putWorkflow(User operator, String projectName, String name, String desc, String proxyUser, String queue, String data, MultipartFile file, String extras) { ProjectFlow projectFlow = flowDao.projectFlowFindByPorjectNameAndName(projectName, name); if (projectFlow == null) { return createWorkflow(operator, projectName, name, desc, proxyUser, queue, data, file, extras, null); } return patchWorkflow(operator, projectName, name, desc, proxyUser, queue, data, file, extras); } /** * ????? \"w\" ?? * <p> * ???null? * </p> * * @param operator * @param projectName * @param name * @param desc * @param proxyUser * @param queue * @param data * @param file * @return */ public ProjectFlow patchWorkflow(User operator, String projectName, String name, String desc, String proxyUser, String queue, String data, MultipartFile file, String extras) { verifyDesc(desc); verifyExtras(extras); Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasWritePerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" write permission", operator.getName(), project.getName()); } // proxyUser ??? if (StringUtils.isNotEmpty(proxyUser)) { verifyProxyUser(operator.getProxyUserList(), proxyUser); } ProjectFlow projectFlow = flowDao.projectFlowfindByName(project.getId(), name); if (projectFlow == null) { logger.error("Not found project flow {} in project {}", name, project.getName()); throw new NotFoundException("Not found project flow \"{0}\" in project \"{1}\"", name, project.getName()); } Date now = new Date(); // ? WorkflowData workflowData = workflowDataDes(data, file, name, project); if (workflowData != null) { if (!CollectionUtils.isEmpty(workflowData.getUserDefParams())) { projectFlow.setUserDefinedParamList(workflowData.getUserDefParams()); } List<WorkflowNodeDto> workflowNodeDTOList = workflowData.getNodes(); if (CollectionUtils.isNotEmpty(workflowNodeDTOList)) { // if (graphHasCycle(workflowNodeDTOList)) { logger.error("Graph has cycle"); throw new BadRequestException("Flow node has cycle"); } // parameter for (WorkflowNodeDto workflowNodeDTO : workflowNodeDTOList) { if (!flowNodeParamCheck(workflowNodeDTO.getParameter(), workflowNodeDTO.getType())) { logger.error("Flow node {} parameter invalid", workflowNodeDTO.getName()); throw new BadRequestException("workflow node parameter not valid"); } // ?? verifyExtras(workflowNodeDTO.getExtras()); } // flowNode List<FlowNode> flowNodeList = new ArrayList<>(); for (WorkflowNodeDto workflowNodeDTO : workflowNodeDTOList) { try { flowNodeList.add(workflowNodeDTO.convertFlowNode()); } catch (JsonProcessingException e) { logger.error("workflow node dto convert flowNode error", e); throw new BadRequestException("workflow node parameter not valid"); } } projectFlow.setFlowsNodes(flowNodeList); } } if (StringUtils.isNotEmpty(extras)) { projectFlow.setExtras(extras); } if (StringUtils.isNotEmpty(desc)) { projectFlow.setDesc(desc); } projectFlow.setModifyTime(now); if (StringUtils.isNotEmpty(proxyUser)) { projectFlow.setProxyUser(proxyUser); } if (StringUtils.isNotEmpty(queue)) { projectFlow.setQueue(queue); } projectFlow.setOwnerId(operator.getId()); projectFlow.setOwner(operator.getName()); try { flowDao.modifyProjectFlow(projectFlow); } catch (Exception e) { logger.error("Workflow modify has error", e); throw new ServerErrorException("Workflow modify has error", e); } return projectFlow; } /** * ?? * * @param operator * @param projectName * @param srcWorkflowName * @param destWorkflowName * @return */ public ProjectFlow postWorkflowCopy(User operator, String projectName, String srcWorkflowName, String destWorkflowName) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasWritePerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" write permission", operator.getName(), project.getName()); } ProjectFlow srcProjectFlow = flowDao.projectFlowfindByName(project.getId(), srcWorkflowName); if (srcProjectFlow == null) { logger.error("Not found project flow {} in project {}", srcWorkflowName, project.getName()); throw new NotFoundException("Not found project flow \"{0}\" in project \"{1}\"", srcWorkflowName, project.getName()); } String data = JsonUtil.toJsonString(new WorkflowData(srcProjectFlow.getFlowsNodes(), srcProjectFlow.getUserDefinedParamList(), FlowNode.class)); // ?? String srcHdfsFilename = BaseConfig.getHdfsWorkflowFilename(project.getId(), srcWorkflowName); String destHdfsFilename = BaseConfig.getHdfsWorkflowFilename(project.getId(), destWorkflowName); logger.info("try copy workflow file {} to workflow file {}", srcHdfsFilename, destHdfsFilename); try { if (HdfsClient.getInstance().exists(srcHdfsFilename)) { HdfsClient.getInstance().copy(srcHdfsFilename, destHdfsFilename, false, true); } else { logger.info("workflow file {} not exists no copy required", srcHdfsFilename); } } catch (IOException e) { logger.error("copy hdfs file error", e); throw new ServerErrorException("copy hdfs file error"); } return putWorkflow(operator, projectName, destWorkflowName, srcProjectFlow.getDesc(), srcProjectFlow.getProxyUser(), srcProjectFlow.getQueue(), data, null, srcProjectFlow.getExtras()); } /** * ? * * @param operator * @param projectName * @param name */ public void deleteProjectFlow(User operator, String projectName, String name) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // ?? if (!projectService.hasWritePerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" write permission", operator.getName(), project.getName()); } ProjectFlow projectFlow = flowDao.projectFlowfindByName(project.getId(), name); if (projectFlow == null) { logger.error("Not found project flow {} in project {}", name, project.getName()); throw new NotFoundException("Not found project flow \"{0}\" in project \"{1}\"", name, project.getName()); } // ? flowDao.deleteWorkflow(projectFlow.getId()); // ??? String hdfsFilename = BaseConfig.getHdfsWorkflowFilename(project.getId(), name); HdfsClient.getInstance().delete(hdfsFilename, true); } /** * ?? * * @param operator * @param projectName * @param queue * @param proxyUser */ public void modifyWorkflowConf(User operator, String projectName, String queue, String proxyUser) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasWritePerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" write permission", operator.getName(), project.getName()); } projectFlowMapper.updateProjectConf(project.getId(), queue, proxyUser); } /** * ? * * @param operator * @param projectName * @return */ public List<ProjectFlow> queryAllProjectFlow(User operator, String projectName) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasReadPerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" read permission", operator.getName(), project.getName()); } return flowDao.projectFlowFindByProject(project.getId()); } /** * ?? * * @param operator * @param projectName * @param name * @return */ public List<ProjectFlow> queryProjectFlow(User operator, String projectName, String name) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasReadPerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" read permission", operator.getName(), project.getName()); } // projectFlow ProjectFlow projectFlow = flowDao.projectFlowfindByName(project.getId(), name); if (projectFlow == null) { throw new NotFoundException("Not found Workflow \"{0}\"", name); } return Arrays.asList(projectFlow); } /** * @param operator * @param projectName * @param name * @return */ public org.springframework.core.io.Resource downloadProjectFlowFile(User operator, String projectName, String name) { Project project = projectMapper.queryByName(projectName); if (project == null) { logger.error("Project does not exist: {}", projectName); throw new NotFoundException("Not found project \"{0}\"", projectName); } // project ?? if (!projectService.hasReadPerm(operator.getId(), project)) { logger.error("User {} has no right permission for the project {}", operator.getName(), project.getName()); throw new PermissionException("User \"{0}\" is not has project \"{1}\" read permission", operator.getName(), project.getName()); } // ? hdfs logger.info("try download workflow {} file from hdfs", name); String localFilename = BaseConfig.getLocalWorkflowFilename(project.getId(), UUID.randomUUID().toString()); String hdfsFilename = BaseConfig.getHdfsWorkflowFilename(project.getId(), name); logger.info("download hdfs {} to local {}", hdfsFilename, localFilename); try { HdfsClient.getInstance().copyHdfsToLocal(hdfsFilename, localFilename, false, true); } catch (Exception e) { logger.error("try download workflow {} file error", e); } org.springframework.core.io.Resource file = fileSystemStorageService.loadAsResource(localFilename); if (file != null) { return file; } // ? logger.info("try download workflow {} file from db", name); ProjectFlow projectFlow = flowDao.projectFlowfindByName(project.getId(), name); try { String json = JsonUtil.toJsonString(new WorkflowData(projectFlow.getFlowsNodes(), projectFlow.getUserDefinedParamList(), FlowNode.class)); InputStreamResource resource = new InputStreamResource(new FileInputStream(json)); return resource; } catch (Exception e) { logger.error("download workflow file from db error", e); throw new ServerErrorException("download workflow file from db error"); } } /** * ? data ??? * * @param data * @param file * @param workflowName * @param project * @return */ private WorkflowData workflowDataDes(String data, MultipartFile file, String workflowName, Project project) { WorkflowData workflowData = null; if (file != null && !file.isEmpty()) { // ? String filename = UUID.randomUUID().toString(); // , ".zip" String localFilename = BaseConfig.getLocalWorkflowFilename(project.getId(), filename); // ? String localExtractDir = BaseConfig.getLocalWorkflowExtractDir(project.getId(), filename); // hdfs String hdfsFilename = BaseConfig.getHdfsWorkflowFilename(project.getId(), workflowName); try { // logger.info("save workflow file {} to local {}", workflowName, localFilename); fileSystemStorageService.store(file, localFilename); // ? ZipFile zipFile = new ZipFile(localFilename); logger.info("ext file {} to {}", localFilename, localExtractDir); zipFile.extractAll(localExtractDir); String workflowJsonPath = MessageFormat.format("{0}/{1}", localExtractDir, WorkflowJson); logger.info("Start reader workflow.json: {}", workflowJsonPath); String jsonString = fileSystemStorageService.readFileToString(workflowJsonPath); workflowData = JsonUtil.parseObject(jsonString, WorkflowData.class); logger.info("Finish reader workflow.json!"); // HDFS if (workflowData != null) { logger.info("update workflow local file {} to hdfs {}", localFilename, hdfsFilename); HdfsClient.getInstance().copyLocalToHdfs(localFilename, hdfsFilename, true, true); } } catch (ZipException e) { logger.error("ext file error", e); return null; } catch (IOException e) { logger.error("read workflow.json error", e); return null; } catch (Exception e) { logger.error("workflow file process error", e); return null; } } else if (data != null) { workflowData = JsonUtil.parseObject(data, WorkflowData.class); } return workflowData; } /** * * * @param workflowNodeResponseList * @return */ private boolean graphHasCycle(List<WorkflowNodeDto> workflowNodeResponseList) { Graph<String, WorkflowNodeDto, String> graph = new Graph<>(); // for (WorkflowNodeDto workflowNodeResponse : workflowNodeResponseList) { graph.addVertex(workflowNodeResponse.getName(), workflowNodeResponse); } // for (WorkflowNodeDto workflowNodeResponse : workflowNodeResponseList) { if (CollectionUtils.isNotEmpty(workflowNodeResponse.getDep())) { for (String dep : workflowNodeResponse.getDep()) { graph.addEdge(dep, workflowNodeResponse.getName()); } } } return graph.hasCycle(); } }