Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tez.service.impl; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import com.google.common.base.Preconditions; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezTaskUmbilicalProtocol; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.TokenCache; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.hadoop.shim.DefaultHadoopShim; import org.apache.tez.runtime.api.impl.TaskSpec; import org.apache.tez.runtime.task.TaskReporter; import org.apache.tez.runtime.task.TaskRunner2Result; import org.apache.tez.runtime.task.TezTaskRunner2; import org.apache.tez.service.ContainerRunner; import org.apache.tez.dag.api.TezConstants; import org.apache.tez.runtime.api.ExecutionContext; import org.apache.tez.runtime.api.impl.ExecutionContextImpl; import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl; import org.apache.tez.runtime.task.TezChild; import org.apache.tez.runtime.task.TezChild.ContainerExecutionResult; import org.apache.tez.shufflehandler.ShuffleHandler; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkRequestProto; import org.apache.tez.util.ProtoConverters; import org.apache.tez.util.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ContainerRunnerImpl extends AbstractService implements ContainerRunner { private static final Logger LOG = LoggerFactory.getLogger(ContainerRunnerImpl.class); public static final String DAG_NAME_INSTRUMENTED_FAILURES = "InstrumentedFailures"; private final ListeningExecutorService executorService; private final AtomicReference<InetSocketAddress> localAddress; private final String[] localDirsBase; private final Map<String, String> localEnv = new HashMap<String, String>(); private volatile FileSystem localFs; private final long memoryPerExecutor; // TODO Support for removing queued containers, interrupting / killing specific containers - when preemption is supported public ContainerRunnerImpl(int numExecutors, String[] localDirsBase, AtomicReference<InetSocketAddress> localAddress, long totalMemoryAvailableBytes) { super("ContainerRunnerImpl"); Preconditions.checkState(numExecutors > 0, "Invalid number of executors: " + numExecutors + ". Must be > 0"); this.localDirsBase = localDirsBase; this.localAddress = localAddress; ExecutorService raw = Executors.newFixedThreadPool(numExecutors, new ThreadFactoryBuilder().setNameFormat("ContainerExecutor %d").build()); this.executorService = MoreExecutors.listeningDecorator(raw); // 80% of memory considered for accounted buffers. Rest for objects. // TODO Tune this based on the available size. this.memoryPerExecutor = (long) (totalMemoryAvailableBytes * 0.8 / (float) numExecutors); LOG.info("ContainerRunnerImpl config: " + "memoryPerExecutorDerived=" + memoryPerExecutor + ", numExecutors=" + numExecutors); } @Override public void serviceInit(Configuration conf) { try { localFs = FileSystem.getLocal(conf); } catch (IOException e) { throw new RuntimeException("Failed to setup local filesystem instance", e); } } @Override public void serviceStart() { } public void setShufflePort(int shufflePort) { AuxiliaryServiceHelper.setServiceDataIntoEnv(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(shufflePort), localEnv); } @Override protected void serviceStop() throws Exception { super.serviceStop(); } // TODO Move this into a utilities class private static String createAppSpecificLocalDir(String baseDir, String applicationIdString, String user) { return baseDir + File.separator + "usercache" + File.separator + user + File.separator + "appcache" + File.separator + applicationIdString; } /** * Submit a container which is ready for running. * The regular pull mechanism will be used to fetch work from the AM * @param request * @throws TezException */ @Override public void queueContainer(RunContainerRequestProto request) throws TezException { LOG.info("Queuing container for execution: " + request); Map<String, String> env = new HashMap<String, String>(); env.putAll(localEnv); env.put(ApplicationConstants.Environment.USER.name(), request.getUser()); String[] localDirs = new String[localDirsBase.length]; // Setup up local dirs to be application specific, and create them. for (int i = 0; i < localDirsBase.length; i++) { localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser()); try { localFs.mkdirs(new Path(localDirs[i])); } catch (IOException e) { throw new TezException(e); } } LOG.info("Dirs for {} are {}", request.getContainerIdString(), Arrays.toString(localDirs)); // Setup workingDir. This is otherwise setup as Environment.PWD // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream) String workingDir = localDirs[0]; Credentials credentials = new Credentials(); DataInputBuffer dib = new DataInputBuffer(); byte[] tokenBytes = request.getCredentialsBinary().toByteArray(); dib.reset(tokenBytes, tokenBytes.length); try { credentials.readTokenStorageStream(dib); } catch (IOException e) { throw new TezException(e); } Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes. LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString()); ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser()); ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor); ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable); Futures.addCallback(future, new ContainerRunnerCallback(request, callable)); } /** * Submit an entire work unit - containerId + TaskSpec. * This is intended for a task push from the AM * * @param request * @throws org.apache.tez.dag.api.TezException */ @Override public void submitWork(SubmitWorkRequestProto request) throws TezException { LOG.info("Queuing work for execution: " + request); checkAndThrowExceptionForTests(request); Map<String, String> env = new HashMap<String, String>(); env.putAll(localEnv); env.put(ApplicationConstants.Environment.USER.name(), request.getUser()); String[] localDirs = new String[localDirsBase.length]; // Setup up local dirs to be application specific, and create them. for (int i = 0; i < localDirsBase.length; i++) { localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser()); try { localFs.mkdirs(new Path(localDirs[i])); } catch (IOException e) { throw new TezException(e); } } if (LOG.isDebugEnabled()) { LOG.debug("Dirs are: " + Arrays.toString(localDirs)); } // Setup workingDir. This is otherwise setup as Environment.PWD // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream) String workingDir = localDirs[0]; Credentials credentials = new Credentials(); DataInputBuffer dib = new DataInputBuffer(); byte[] tokenBytes = request.getCredentialsBinary().toByteArray(); dib.reset(tokenBytes, tokenBytes.length); try { credentials.readTokenStorageStream(dib); } catch (IOException e) { throw new TezException(e); } Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes. LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString()); ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser()); TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor); ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable); Futures.addCallback(future, new TaskRunnerCallback(request, callable)); } static class ContainerRunnerCallable implements Callable<ContainerExecutionResult> { private final RunContainerRequestProto request; private final Configuration conf; private final String workingDir; private final String[] localDirs; private final Map<String, String> envMap; private final String pid = null; private final ObjectRegistryImpl objectRegistry; private final ExecutionContext executionContext; private final Credentials credentials; private final long memoryAvailable; private volatile TezChild tezChild; ContainerRunnerCallable(RunContainerRequestProto request, Configuration conf, ExecutionContext executionContext, Map<String, String> envMap, String[] localDirs, String workingDir, Credentials credentials, long memoryAvailable) { this.request = request; this.conf = conf; this.executionContext = executionContext; this.envMap = envMap; this.workingDir = workingDir; this.localDirs = localDirs; this.objectRegistry = new ObjectRegistryImpl(); this.credentials = credentials; this.memoryAvailable = memoryAvailable; } @Override public ContainerExecutionResult call() throws Exception { StopWatch sw = new StopWatch().start(); tezChild = new TezChild(conf, request.getAmHost(), request.getAmPort(), request.getContainerIdString(), request.getTokenIdentifier(), request.getAppAttemptNumber(), workingDir, localDirs, envMap, objectRegistry, pid, executionContext, credentials, memoryAvailable, request.getUser(), null, false, new DefaultHadoopShim()); ContainerExecutionResult result = tezChild.run(); LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" + sw.stop().now(TimeUnit.MILLISECONDS)); return result; } public TezChild getTezChild() { return this.tezChild; } } final class ContainerRunnerCallback implements FutureCallback<ContainerExecutionResult> { private final RunContainerRequestProto request; private final ContainerRunnerCallable containerRunnerCallable; ContainerRunnerCallback(RunContainerRequestProto request, ContainerRunnerCallable containerRunnerCallable) { this.request = request; this.containerRunnerCallable = containerRunnerCallable; } // TODO Proper error handling @Override public void onSuccess(ContainerExecutionResult result) { switch (result.getExitStatus()) { case SUCCESS: LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString()); break; case EXECUTION_FAILURE: LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), result.getThrowable()); break; case INTERRUPTED: LOG.info("Interrupted while running: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), result.getThrowable()); break; case ASKED_TO_DIE: LOG.info("Asked to die while running: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString()); break; } } @Override public void onFailure(Throwable t) { LOG.error("TezChild execution failed for : " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), t); TezChild tezChild = containerRunnerCallable.getTezChild(); if (tezChild != null) { tezChild.shutdown(); } } } static class TaskRunnerCallable implements Callable<ContainerExecutionResult> { private final SubmitWorkRequestProto request; private final Configuration conf; private final String workingDir; private final String[] localDirs; private final Map<String, String> envMap; private final String pid = null; private final ObjectRegistryImpl objectRegistry; private final ExecutionContext executionContext; private final Credentials credentials; private final long memoryAvailable; private final ListeningExecutorService executor; private volatile TezTaskRunner2 taskRunner; private volatile TaskReporter taskReporter; private TezTaskUmbilicalProtocol umbilical; TaskRunnerCallable(SubmitWorkRequestProto request, Configuration conf, ExecutionContext executionContext, Map<String, String> envMap, String[] localDirs, String workingDir, Credentials credentials, long memoryAvailable) { this.request = request; this.conf = conf; this.executionContext = executionContext; this.envMap = envMap; this.workingDir = workingDir; this.localDirs = localDirs; this.objectRegistry = new ObjectRegistryImpl(); this.credentials = credentials; this.memoryAvailable = memoryAvailable; // TODO This executor seems unnecessary. Here and TezChild ExecutorService executorReal = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true) .setNameFormat("TezTaskRunner_" + request.getTaskSpec().getTaskAttemptIdString()) .build()); executor = MoreExecutors.listeningDecorator(executorReal); } @Override public ContainerExecutionResult call() throws Exception { // TODO Consolidate this code with TezChild. StopWatch sw = new StopWatch().start(); UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser()); taskUgi.addCredentials(credentials); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>(); serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, TezCommonUtils.convertJobTokenToBytes(jobToken)); Multimap<String, String> startedInputsMap = HashMultimap.create(); UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(request.getTokenIdentifier()); final InetSocketAddress address = NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort()); SecurityUtil.setTokenService(jobToken, address); taskOwner.addToken(jobToken); umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() { @Override public TezTaskUmbilicalProtocol run() throws Exception { return RPC.getProxy(TezTaskUmbilicalProtocol.class, TezTaskUmbilicalProtocol.versionID, address, conf); } }); // TODO Stop reading this on each request. taskReporter = new TaskReporter(umbilical, conf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT), conf.getLong(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT), conf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT, TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT), new AtomicLong(0), request.getContainerIdString()); taskRunner = new TezTaskRunner2(conf, taskUgi, localDirs, ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()), request.getAppAttemptNumber(), serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memoryAvailable, false, new DefaultHadoopShim()); boolean shouldDie; try { TaskRunner2Result result = taskRunner.run(); LOG.info("TaskRunner2Result: {}", result); shouldDie = result.isContainerShutdownRequested(); if (shouldDie) { LOG.info("Got a shouldDie notification via heartbeats. Shutting down"); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM"); } if (result.getError() != null) { Throwable e = result.getError(); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, e, "TaskExecutionFailure: " + e.getMessage()); } } finally { FileSystem.closeAllForUGI(taskUgi); } LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" + sw.stop().now(TimeUnit.MILLISECONDS)); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, null); } public void shutdown() { executor.shutdownNow(); if (taskReporter != null) { taskReporter.shutdown(); } if (umbilical != null) { RPC.stopProxy(umbilical); } } } final class TaskRunnerCallback implements FutureCallback<ContainerExecutionResult> { private final SubmitWorkRequestProto request; private final TaskRunnerCallable taskRunnerCallable; TaskRunnerCallback(SubmitWorkRequestProto request, TaskRunnerCallable containerRunnerCallable) { this.request = request; this.taskRunnerCallable = containerRunnerCallable; } // TODO Proper error handling @Override public void onSuccess(ContainerExecutionResult result) { switch (result.getExitStatus()) { case SUCCESS: LOG.info("Successfully finished: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString()); break; case EXECUTION_FAILURE: LOG.info("Failed to run: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), result.getThrowable()); break; case INTERRUPTED: LOG.info("Interrupted while running: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), result.getThrowable()); break; case ASKED_TO_DIE: LOG.info("Asked to die while running: " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString()); break; } taskRunnerCallable.shutdown(); } @Override public void onFailure(Throwable t) { LOG.error("TezTaskRunner execution failed for : " + request.getApplicationIdString() + ", containerId=" + request.getContainerIdString(), t); taskRunnerCallable.shutdown(); } } private void checkAndThrowExceptionForTests(SubmitWorkRequestProto request) throws TezException { if (!request.getTaskSpec().getDagName().equals(DAG_NAME_INSTRUMENTED_FAILURES)) { return; } TaskSpec taskSpec = ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()); if (taskSpec.getTaskAttemptID().getTaskID().getId() == 0 && taskSpec.getTaskAttemptID().getId() == 0) { LOG.info("Simulating Rejected work"); throw new RejectedExecutionException( "Simulating Rejected work for taskAttemptId=" + taskSpec.getTaskAttemptID()); } else if (taskSpec.getTaskAttemptID().getTaskID().getId() == 1 && taskSpec.getTaskAttemptID().getId() == 0) { LOG.info("Simulating Task Setup Failure during launch"); throw new TezException( "Simulating Task Setup Failure during launch for taskAttemptId=" + taskSpec.getTaskAttemptID()); } } }