Java tutorial
/* * Copyright (C) 2008 Yohan Liyanage. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.nebulaframework.grid.cluster.manager.services.jobs.unbounded; import java.io.Serializable; import javax.jms.ConnectionFactory; import javax.jms.JMSException; import javax.jms.Message; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nebulaframework.core.job.GridJobState; import org.nebulaframework.core.job.ResultCallback; import org.nebulaframework.core.job.annotations.unbounded.UnboundedProcessingSettings; import org.nebulaframework.core.job.exceptions.InvalidResultException; import org.nebulaframework.core.job.exceptions.SecurityViolationException; import org.nebulaframework.core.job.unbounded.UnboundedGridJob; import org.nebulaframework.core.job.unbounded.UnboundedSettingsAware; import org.nebulaframework.core.task.GridTask; import org.nebulaframework.core.task.GridTaskResult; import org.nebulaframework.grid.cluster.manager.ClusterManager; import org.nebulaframework.grid.cluster.manager.services.jobs.GridJobProfile; import org.nebulaframework.grid.cluster.manager.services.jobs.InternalClusterJobService; import org.nebulaframework.grid.cluster.manager.services.jobs.ResultCollectionSupport; import org.nebulaframework.grid.cluster.manager.support.CleanUpSupport; import org.nebulaframework.util.jms.JMSNamingSupport; import org.springframework.jms.core.JmsTemplate; import org.springframework.jms.core.MessagePostProcessor; import org.springframework.jms.listener.DefaultMessageListenerContainer; import org.springframework.jms.listener.adapter.MessageListenerAdapter; import org.springframework.util.Assert; /** * Manages the execution of {@code UnboundedGridJob}s. For each active * {@code UnboundedGridJob} an instance of this class will be created by * the {@code ClusterManager}. * <p> * This class is responsible for enqueue tasks for the {@code UnboundedGridJob} * and also to retrieve results for enqueued tasks. Furthermore, it invokes * the {@link ResultCallback}s for intermediate results, if such a callback * is available. * * @author Yohan Liyanage * @version 1.0 * * @see UnboundedGridJob */ public class UnboundedJobProcessor extends ResultCollectionSupport { private static Log log = LogFactory.getLog(UnboundedJobProcessor.class); private UnboundedGridJob<?> job; // GridJob private boolean canceled = false; private JmsTemplate jmsTemplate; private ConnectionFactory connectionFactory; private InternalClusterJobService jobService; private DefaultMessageListenerContainer container; /* -- Default Processing Settings -- */ /** Maximum number of tasks in TaskQueue at a given time, without slowing * task generation */ private int maxTaskConstant = 100; /** * Factor (time) by which the task generation is slowed per task which is over * maxTaskConstant (in milliseconds) */ private int reductionFactorConstant = 50; /** * Indicates whether to stop task generation if a null task is returned * after invoking task() method on UnboundedGridJob */ private boolean stopOnNullTask = true; /** * Indicates whether the tasks generated for the current UnboundedGridJob * are mutually exclusive, which can be used to increase performance * and resource utilization */ private boolean mutuallyExclusiveTasks = false; /** * Constructs a {@code UnboundedJobProcessor} which * will manage the execution of {@code UnboundedGridJob} * represented by the {@code GridJobProfile}. * * @param profile * @param connectionFactory * @param jobService */ public UnboundedJobProcessor(GridJobProfile profile) { super(); // Validate Arguments Assert.notNull(profile); if (!(profile.getJob() instanceof UnboundedGridJob<?>)) { throw new IllegalArgumentException("GridJob is not a UnboundedGridJob"); } this.profile = profile; this.job = (UnboundedGridJob<?>) profile.getJob(); // Use Reflection to extract any processing instructions extractProcessingSettings(job); this.connectionFactory = ClusterManager.getInstance().getConnectionFactory(); this.jobService = ClusterManager.getInstance().getJobService(); } /** * Extracts processing settings for a given {@code UnboundedGridJob} * from annotations of the class, if available. * * @param job {@code UnboundedGridJob} job */ private void extractProcessingSettings(UnboundedGridJob<?> job) { if (job instanceof UnboundedSettingsAware) { UnboundedSettingsAware settings = (UnboundedSettingsAware) job; this.maxTaskConstant = settings.maxTasksInQueue(); this.reductionFactorConstant = settings.reductionFactor(); this.stopOnNullTask = settings.stopOnNullTask(); this.mutuallyExclusiveTasks = settings.mutuallyExclusiveTasks(); return; } Class<?> clazz = job.getClass(); // Get reference to ProcessingSettings annotation UnboundedProcessingSettings settings = clazz.getAnnotation(UnboundedProcessingSettings.class); // If not annotated, return if (settings == null) return; // If available, retrieve settings this.maxTaskConstant = settings.maxTasksInQueue(); this.reductionFactorConstant = settings.reductionFactor(); this.stopOnNullTask = settings.stopOnNullTask(); this.mutuallyExclusiveTasks = settings.mutuallyExclusiveTasks(); log.debug("[UnboundedJobProcessor] Using Custom Processing Settings from Annotation"); } /** * Starts this {@code UnboundedJobProcessor} instance by * initializing JMS resources and starting task generation. */ public void start() { initialize(); generateTasks(); } /** * Initializes this {@code UnboundedJobProcessor}'s JMS * resources. */ private void initialize() { initializeResultListener(); initializeTaskWritier(); } /** * Register's {@link #onResult(GridTaskResult)} method as the listener * method for ResultQueue. */ private void initializeResultListener() { MessageListenerAdapter adapter = new MessageListenerAdapter(this); adapter.setDefaultListenerMethod("onResult"); container = new DefaultMessageListenerContainer(); container.setConnectionFactory(connectionFactory); container.setDestinationName(JMSNamingSupport.getResultQueueName(profile.getJobId())); container.setMessageListener(adapter); container.afterPropertiesSet(); // Clean Up Hook CleanUpSupport.shutdownContainerWhenFinished(profile.getJobId(), container); } /** * Creates the {@code JmsTemplate} used to write * tasks to TaskQueue. */ private void initializeTaskWritier() { this.jmsTemplate = new JmsTemplate(connectionFactory); } /** * Starts generation of {@code GridTask}s by repetitively * invoking the {@link UnboundedGridJob#task()} method. */ private void generateTasks() { log.info("[UnboundedJobProcessor] Started Generating Tasks"); // Start on a new thread new Thread(new Runnable() { public void run() { // Start Tracker profile.getTaskTracker().start(); // Update State profile.getFuture().setState(GridJobState.EXECUTING); int taskId = 0; while (true) { // If Job is Canceled, Stop if (isCanceled()) { log.debug("[UnboundedJobProcessor] Task was cancelled. Stopping Task Generation"); break; } GridTask<?> task = null; try { // Get next task to be enqueued task = job.task(); } catch (SecurityException e) { log.error("[UnboundedJobProcessor] Security Violation while invoking task()", e); log.warn("[UnboundedJobProcessor] Stopping Task Generation"); // Update Future profile.getFuture().setState(GridJobState.FAILED); profile.getFuture().setException(e); // Stop Job Execution stopJob(); break; } catch (Exception e) { log.warn("[UnboundedJobProcessor] Exception while invoking task()", e); log.warn("[UnboundedJobProcessor] Stopping Task Generation"); // Update Future profile.getFuture().setState(GridJobState.FAILED); profile.getFuture().setException(e); // Stop Job Execution stopJob(); break; } // If Task was null if (task == null) { if (stopOnNullTask) { log.info("[UnboundedJobProcessor] Task was Null. Stopping Task Generation"); break; } else { log.warn("[UnboundedJobProcessor] Task was Null. Ignoring"); continue; } } // Enqueue Task enqueueTask(profile.getJobId(), ++taskId, task); if (!mutuallyExclusiveTasks) { // If tasks are not mutually exclusive keep track of real task profile.addTask(taskId, task); } else { // If tasks are mutually exclusive, save memory by storing a null profile.addTask(taskId, null); } // Slow down for some duration if more than 'maxTaskConstant' tasks are // there to ensure TaskQueue won't overload try { if (profile.getTaskCount() > maxTaskConstant) { Thread.sleep(profile.getTaskCount() * reductionFactorConstant); } } catch (InterruptedException e) { log.error(e); } } waitIfNeeded(); // Stop Job (Exception / Task Null) stopJob(); } }).start(); } /** * Enqueues a given Task with in the {@code TaskQueue}. * * @param jobId * String JobId * @param taskId * int TaskId (Sequence Number of Task) * @param task * {@code GridTask} task */ private void enqueueTask(final String jobId, final int taskId, GridTask<?> task) { String queueName = JMSNamingSupport.getTaskQueueName(jobId); // Post Process to include Meta Data MessagePostProcessor postProcessor = new MessagePostProcessor() { public Message postProcessMessage(Message message) throws JMSException { // Set Correlation ID to Job Id message.setJMSCorrelationID(jobId); // Put taskId as a property message.setIntProperty("taskId", taskId); log.debug("Enqueued Task : " + taskId); return message; } }; // Send GridTask as a JMS Object Message to TaskQueue jmsTemplate.convertAndSend(queueName, task, postProcessor); // Update Task Tracker profile.getTaskTracker().taskEnqueued(taskId); } /** * Re-enqueues the {@code GridTask} denoted by {@code taskId}. * * @param taskId {@code GridTask} Id */ public void reEnqueueTask(final int taskId) { log.debug("Re-enqueueing Task : " + taskId); if (!mutuallyExclusiveTasks) { enqueueTask(profile.getJobId(), taskId, job.task()); } else { GridTask<?> task = profile.getTask(taskId); // If Task Not in Profile (Completed) if (task == null) { log.debug("[Processor] Unable to re-enqueue, task possibly complete" + profile.getJobId() + "|" + taskId); return; } enqueueTask(profile.getJobId(), taskId, task); } } /** * Stops execution of the {@code UnboundedGridJob}, * and notifies the workers. */ protected void stopJob() { log.info("[UnboundedJobProcessor] Stopping Job Execution"); // Notify Workers jobService.notifyJobEnd(profile.getJobId()); // Update Future and return Result (null for unbounded) profile.getFuture().setResult(null); if (!profile.getFuture().isJobFinished()) { profile.getFuture().setState(GridJobState.COMPLETE); } // Destroy this instance destroy(); } /** * Invoked by the JMS Message Listener Container when a result * is available in the ResultQueue. * * @param taskResult Result of Task */ public void onResult(final GridTaskResult taskResult) { // Result is Valid / Complete if (taskResult.isComplete()) { log.debug("[UnboundedJobProcessor] Received Result : Task " + taskResult.getTaskId()); // Update Tracker profile.getTaskTracker().resultReceived(taskResult.getTaskId(), taskResult.getExecutionTime()); // Post Process Result Serializable result; try { result = job.processResult(taskResult.getResult()); } catch (InvalidResultException e) { // Result was invalid, re-enqueue log.debug("[UnboundedJobProcessor] Invalid Result Exception"); // Update Profile profile.failedTaskReceived(); // Add Failure Trace addFailureTrace(taskResult.getWorkerId()); reEnqueueTask(taskResult.getTaskId()); return; } catch (SecurityException e) { log.error("[UnboundedJobProcessor] Security Violation while Processing Result", e); log.warn("[UnboundedJobProcessor] Stopping Job Execution"); waitIfNeeded(); // Update Future profile.getFuture().setState(GridJobState.FAILED); profile.getFuture().setException(e); // Stop Job Execution stopJob(); return; } catch (Exception e) { log.warn("[UnboundedJobProcessor] Exception while Processing Result", e); log.warn("[UnboundedJobProcessor] Stopping Job Execution"); waitIfNeeded(); // Update Future profile.getFuture().setState(GridJobState.FAILED); profile.getFuture().setException(e); // Stop Job Execution stopJob(); return; } // Fire intermediate results callback profile.fireCallback(result); // Clear Failure Traces clearFailureTrace(taskResult.getWorkerId()); // Task completed, remove it from TaskMap // Add Dummy Place-holder for Result List profile.addResultAndRemoveTask(taskResult.getTaskId(), null); } else { // Result Not Valid / Exception // Check for Security Violations (Fails Job) if (taskResult.getException() instanceof SecurityException) { log.error("[UnboundedJobProcessor] Security Violation detected. Terminating GridJob" + profile.getJobId()); waitIfNeeded(); // Fail the Job profile.getFuture().fail( new SecurityViolationException("Security Violation Detected", taskResult.getException())); // Stop Result Collector destroy(); return; } // Update Profile profile.failedTaskReceived(); // Add Failure Trace addFailureTrace(taskResult.getWorkerId()); log.warn("[UnboundedJobProcessor] Result Failed [" + taskResult.getTaskId() + "], ReEnqueueing - " + taskResult.getException()); // Request re-enqueue of Task reEnqueueTask(taskResult.getTaskId()); } } /** * Shutdowns the JMS Message Listener to * avoid processing any more results. */ protected void destroy() { if (container != null) container.shutdown(); } /** * {@inheritDoc} */ public boolean cancel() { // Mark that this Job is canceled this.setCanceled(true); // Stop listener try { destroy(); } catch (Exception e) { log.warn("[UnboundedJobProcessor] Unable to shutdown container", e); return false; } return true; } /** * Indicates whether this {@code UnboundedJobProcessor} * is canceled. * * @return if canceled, {@code true}, otherwise {@code false} */ public boolean isCanceled() { return canceled; } /** * Sets the canceled state to the given {@code boolean} value * * @param canceled state */ protected void setCanceled(boolean canceled) { this.canceled = canceled; } /** * Withholds results if execution finished before minimum execution * duration, to avoid thread synchronization issues. */ private void waitIfNeeded() { if (System.currentTimeMillis() - profile.getStartTime() < GridJobProfile.MINIMUM_EXEUCTION_TIME) { try { long duration = GridJobProfile.MINIMUM_EXEUCTION_TIME - (System.currentTimeMillis() - profile.getStartTime()) + 1000; if (duration < 0) return; Thread.sleep(duration); } catch (InterruptedException e) { log.warn("Interrupted", e); } } } }