Java tutorial
/* * Copyright 2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.yarn.batch.support; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.JobParameter; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersIncrementer; import org.springframework.batch.core.configuration.JobRegistry; import org.springframework.batch.core.converter.DefaultJobParametersConverter; import org.springframework.batch.core.converter.JobParametersConverter; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobExecutionNotFailedException; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.launch.JobParametersNotFoundException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.stereotype.Component; import org.springframework.util.PatternMatchUtils; import org.springframework.util.StringUtils; import org.springframework.yarn.batch.event.JobExecutionEvent; import org.springframework.yarn.batch.support.YarnBatchProperties.JobProperties; /** * Utility class to {@link JobLauncher launch} Spring Batch jobs. * Runs all jobs in the surrounding context by default. Can also be used to * launch a specific job by providing a jobName * * @author Dave Syer * @author Janne Valkealahti */ @Component public class YarnJobLauncher implements ApplicationEventPublisherAware { private static final Log log = LogFactory.getLog(YarnJobLauncher.class); private JobParametersConverter converter = new DefaultJobParametersConverter(); private JobLauncher jobLauncher; private JobRegistry jobRegistry; private JobExplorer jobExplorer; private YarnBatchProperties yarnBatchProperties; private Collection<Job> jobs = Collections.emptySet(); private String jobName; private ApplicationEventPublisher publisher; @Override public void setApplicationEventPublisher(ApplicationEventPublisher publisher) { this.publisher = publisher; } /** * Run the jobs. * * @param args the args * @throws JobExecutionException the job execution exception */ public void run(String... args) throws JobExecutionException { log.info("Running default command line with: " + Arrays.asList(args)); launchJobFromProperties(StringUtils.splitArrayElementsIntoProperties(args, "=")); } /** * Run the jobs. * * @param properties the job properties * @throws JobExecutionException the job execution exception */ public void run(Properties properties) throws JobExecutionException { launchJobFromProperties(properties); } /** * Sets the enabled job name. Moreover this name can also be a simple * pattern supported by * {@link PatternMatchUtils#simpleMatch(String, String)} and multiple * patterns can be matched if delimited by a comma. * * @param jobName the new job name */ public void setJobName(String jobName) { this.jobName = jobName; } /** * Gets the enabled job name. * * @return the job name * @see #setJobName(String) */ public String getJobName() { return jobName; } /** * Sets the job launcher. * * @param jobLauncher the new job launcher */ @Autowired public void setJobLauncher(JobLauncher jobLauncher) { this.jobLauncher = jobLauncher; } /** * Gets the job launcher. * * @return the job launcher */ public JobLauncher getJobLauncher() { return jobLauncher; } @Autowired(required = false) public void setJobRegistry(JobRegistry jobRegistry) { this.jobRegistry = jobRegistry; } public JobRegistry getJobRegistry() { return jobRegistry; } @Autowired(required = false) public void setJobParametersConverter(JobParametersConverter converter) { this.converter = converter; } @Autowired(required = false) public void setJobExplorer(JobExplorer jobExplorer) { this.jobExplorer = jobExplorer; } public JobExplorer getJobExplorer() { return jobExplorer; } /** * Sets the jobs. * * @param jobs the new jobs */ @Autowired(required = false) public void setJobs(Collection<Job> jobs) { this.jobs = jobs; } /** * Gets the jobs. * * @return the jobs */ public Collection<Job> getJobs() { return jobs; } @Autowired(required = false) public void setYarnBatchProperties(YarnBatchProperties yarnBatchProperties) { this.yarnBatchProperties = yarnBatchProperties; } /** * Launch jobs using {@link Properties}. * * @param properties the properties * @throws JobExecutionException the job execution exception */ protected void launchJobFromProperties(Properties properties) throws JobExecutionException { JobParameters jobParameters = this.converter.getJobParameters(properties); executeRegisteredJobs(jobParameters); executeLocalJobs(jobParameters); } private void executeRegisteredJobs(JobParameters jobParameters) throws JobExecutionException { if (this.jobRegistry == null) { log.info("No jobRegistry defined, skipping registered jobs"); } if (this.jobRegistry != null && StringUtils.hasText(this.jobName)) { for (String name : this.jobRegistry.getJobNames()) { if (StringUtils.hasText(this.jobName) && jobMatches(this.jobName, name)) { Job job = this.jobRegistry.getJob(name); executeJob(job, jobParameters); } else { log.debug("Skipped registered job: " + name); } } } } private void executeLocalJobs(JobParameters jobParameters) throws JobExecutionException { if (jobs.size() == 0) { log.info("No local jobs defined"); } for (Job job : jobs) { if (StringUtils.hasText(this.jobName) && jobMatches(this.jobName, job.getName())) { log.debug("Executing local job: " + job.getName()); executeJob(job, jobParameters); } else { log.debug("Skipped local job: " + job.getName()); } } } private static boolean jobMatches(String patterns, String name) { for (String pattern : StringUtils.commaDelimitedListToStringArray(patterns)) { if (PatternMatchUtils.simpleMatch(pattern, name)) { return true; } } return false; } protected void executeJob(Job job, JobParameters jobParameters) throws JobExecutionException { String jobIdentifier = job.getName(); JobProperties jobProperties = yarnBatchProperties != null ? yarnBatchProperties.getJobProperties(jobIdentifier) : null; boolean restart = false; // re-create by adding props from a boot JobProperties if (jobProperties != null && jobProperties.getParameters() != null) { log.info("Job parameters from boot properties, parameters" + jobProperties.getParameters()); Properties tmpProperties = new Properties(); Map<String, Object> tmpParameters = jobProperties.getParameters(); tmpProperties.putAll(tmpParameters); JobParameters tmpJobParameters = this.converter.getJobParameters(tmpProperties); Map<String, JobParameter> map1 = new HashMap<String, JobParameter>(tmpJobParameters.getParameters()); map1.putAll(jobParameters.getParameters()); jobParameters = new JobParameters(map1); log.info("Modified jobParameters=" + jobParameters); } if (jobProperties != null && jobProperties.isRestart()) { if (jobExplorer == null) { throw new JobExecutionException( "A JobExplorer must be provided for a restart or start next operation."); } JobExecution jobExecution = getLastFailedJobExecution(jobIdentifier); if (log.isDebugEnabled()) { log.info("Last failed JobExecution: " + jobExecution); } if (jobExecution == null && jobProperties.isFailRestart()) { throw new JobExecutionNotFailedException( "No failed or stopped execution found for job=" + jobIdentifier); } else { log.info("No failed or stopped execution found for job=" + jobIdentifier + ", batch properties flag for failRestart=" + jobProperties.isFailRestart() + " so we don't fail restart."); } if (jobExecution != null) { restart = true; jobParameters = jobExecution.getJobParameters(); } } if (jobProperties != null && jobProperties.isNext() && !restart) { if (jobExplorer == null) { throw new JobExecutionException( "A JobExplorer must be provided for a restart or start next operation."); } JobParameters nextParameters = getNextJobParameters(job, false); Map<String, JobParameter> map = new HashMap<String, JobParameter>(nextParameters.getParameters()); map.putAll(jobParameters.getParameters()); jobParameters = new JobParameters(map); if (log.isDebugEnabled()) { log.info("JobParameter for job=[" + job + "] next=" + nextParameters + " used=" + jobParameters); } } JobExecution execution = this.jobLauncher.run(job, jobParameters); if (this.publisher != null) { this.publisher.publishEvent(new JobExecutionEvent(this, execution)); } } /** * Get next job parameters for a job using {@link JobParametersIncrementer}. * * @param job the job that we need to find the next parameters for * @param fail suppress exception * @return the next job parameters if they can be located * @throws JobParametersNotFoundException if there is a problem */ private JobParameters getNextJobParameters(Job job, boolean fail) throws JobParametersNotFoundException { String jobIdentifier = job.getName(); JobParameters jobParameters; List<JobInstance> lastInstances = jobExplorer.getJobInstances(jobIdentifier, 0, 1); JobParametersIncrementer incrementer = job.getJobParametersIncrementer(); if (incrementer == null) { throw new JobParametersNotFoundException( "No job parameters incrementer found for job=" + jobIdentifier); } if (lastInstances.isEmpty()) { jobParameters = incrementer.getNext(new JobParameters()); if (jobParameters == null) { throw new JobParametersNotFoundException( "No bootstrap parameters found from incrementer for job=" + jobIdentifier); } } else { List<JobExecution> lastExecutions = jobExplorer.getJobExecutions(lastInstances.get(0)); jobParameters = incrementer.getNext(lastExecutions.get(0).getJobParameters()); } return jobParameters; } /** * Gets the last failed job execution. Doesn't returns failed job execution * if there is an successful execution. * * @param jobIdentifier the job identifier * @return the last failed job execution */ private JobExecution getLastFailedJobExecution(String jobIdentifier) { List<JobExecution> jobExecutions = getJobExecutionsWithStatusGreaterThan(jobIdentifier); if (jobExecutions.isEmpty()) { return null; } if (!jobExecutions.get(0).getStatus().isUnsuccessful()) { return null; } else { return jobExecutions.get(0); } } /** * Gets the job executions. * * @param jobIdentifier a job execution id or job name * @return the job executions with status greater than */ private List<JobExecution> getJobExecutionsWithStatusGreaterThan(String jobIdentifier) { int start = 0; int count = 100; List<JobExecution> executions = new ArrayList<JobExecution>(); List<JobInstance> lastInstances = jobExplorer.getJobInstances(jobIdentifier, start, count); while (!lastInstances.isEmpty()) { for (JobInstance jobInstance : lastInstances) { List<JobExecution> jobExecutions = jobExplorer.getJobExecutions(jobInstance); if (jobExecutions == null || jobExecutions.isEmpty()) { continue; } for (JobExecution jobExecution : jobExecutions) { executions.add(jobExecution); } } start += count; lastInstances = jobExplorer.getJobInstances(jobIdentifier, start, count); } return executions; } }