List of usage examples for org.apache.hadoop.yarn.util SystemClock SystemClock
@Deprecated
public SystemClock()
From source file:org.apache.tez.dag.app.dag.impl.TestTaskImpl.java
License:Apache License
@Before public void setup() { conf = new Configuration(); taskAttemptListener = mock(TaskAttemptListener.class); taskHeartbeatHandler = mock(TaskHeartbeatHandler.class); credentials = new Credentials(); clock = new SystemClock(); locationHint = TaskLocationHint.createTaskLocationHint(null, null); appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); dagId = TezDAGID.getInstance(appId, 1); vertexId = TezVertexID.getInstance(dagId, 1); appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); mockContainerId = mock(ContainerId.class); mockContainer = mock(Container.class); mockAMContainer = mock(AMContainer.class); mockNodeId = mock(NodeId.class); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockAMContainer.getContainer()).thenReturn(mockContainer); when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer); taskResource = Resource.newInstance(1024, 1); localResources = new HashMap<String, LocalResource>(); environment = new HashMap<String, String>(); javaOpts = ""; leafVertex = false;/* www . ja v a 2 s. co m*/ containerContext = new ContainerContext(localResources, credentials, environment, javaOpts); Vertex vertex = mock(Vertex.class); eventHandler = new TestEventHandler(); mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskAttemptListener, clock, taskHeartbeatHandler, appContext, leafVertex, locationHint, taskResource, containerContext, vertex); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskRecovery.java
License:Apache License
@Before public void setUp() { dispatcher = new DrainDispatcher(); dispatcher.register(DAGEventType.class, mock(EventHandler.class)); dispatcher.register(VertexEventType.class, mock(EventHandler.class)); dispatcher.register(TaskEventType.class, new TaskEventHandler()); dispatcher.register(TaskAttemptEventType.class, taEventHandler); dispatcher.init(new Configuration()); dispatcher.start();//from ww w . java 2 s .c om vertex = mock(Vertex.class, RETURNS_DEEP_STUBS); when(vertex.getProcessorDescriptor().getClassName()).thenReturn(""); mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))).thenReturn(vertex); task = new TaskImpl(vertexId, 0, dispatcher.getEventHandler(), new Configuration(), mock(TaskAttemptListener.class), new SystemClock(), mock(TaskHeartbeatHandler.class), mockAppContext, false, Resource.newInstance(1, 1), mock(ContainerContext.class), mock(StateChangeNotifier.class)); Map<String, OutputCommitter> committers = new HashMap<String, OutputCommitter>(); committers.put("out1", new TestOutputCommitter(mock(OutputCommitterContext.class), true, false)); when(task.getVertex().getOutputCommitters()).thenReturn(committers); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexRecovery.java
License:Apache License
@Before public void setUp() throws IOException { dispatcher = new DrainDispatcher(); dispatcher.register(DAGEventType.class, mock(EventHandler.class)); vertexEventHandler = new VertexEventHanlder(); dispatcher.register(VertexEventType.class, vertexEventHandler); taskEventHandler = new TaskEventHandler(); dispatcher.register(TaskEventType.class, taskEventHandler); dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventHandler()); dispatcher.init(new Configuration()); dispatcher.start();/* w w w. ja v a2 s . co m*/ mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); DAGPlan dagPlan = createDAGPlan(); dag = new DAGImpl(dagId, new Configuration(), dagPlan, dispatcher.getEventHandler(), mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user, mock(TaskHeartbeatHandler.class), mockAppContext); when(mockAppContext.getCurrentDAG()).thenReturn(dag); dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT)); LOG.info("finish setUp"); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexRecovery.java
License:Apache License
/** * vertex1(New) -> StartRecoveryTransition(SUCCEEDED) * @throws IOException //from w w w. ja va 2 s . com */ @Test(timeout = 5000) public void testRecovery_Desired_SUCCEEDED_OnlySummaryLog() throws IOException { DAGPlan dagPlan = createDAGPlanSingleVertex(); dag = new DAGImpl(dagId, new Configuration(), dagPlan, dispatcher.getEventHandler(), mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user, mock(TaskHeartbeatHandler.class), mockAppContext); when(mockAppContext.getCurrentDAG()).thenReturn(dag); dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT)); VertexImpl vertex1 = (VertexImpl) dag.getVertex("vertex1"); VertexFinishedEvent vertexFinishEvent = new VertexFinishedEvent(); vertexFinishEvent .fromSummaryProtoStream(SummaryEventProto.newBuilder().setDagId(dag.getID().toString()) .setEventType(HistoryEventType.VERTEX_FINISHED.ordinal()).setTimestamp(100L) .setEventPayload(VertexFinishStateProto.newBuilder().setNumTasks(2) .setState(VertexState.SUCCEEDED.ordinal()) .setVertexId(vertex1.getVertexId().toString()).build().toByteString()) .build()); VertexState recoveredState = vertex1.restoreFromEvent(vertexFinishEvent); // numTasks is recovered from summary log assertEquals(2, vertex1.numTasks); assertEquals(VertexState.SUCCEEDED, recoveredState); vertex1.handle(new VertexEventRecoverVertex(vertex1.getVertexId(), VertexState.SUCCEEDED)); dispatcher.await(); assertEquals(VertexState.SUCCEEDED, vertex1.getState()); assertEquals(vertex1.numTasks, vertex1.succeededTaskCount); assertEquals(vertex1.numTasks, vertex1.completedTaskCount); }
From source file:org.apache.tez.dag.app.DAGAppMaster.java
License:Apache License
public static void main(String[] args) { try {/* ww w.ja va 2 s.com*/ Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = System.getenv(Environment.CONTAINER_ID.name()); String nodeHostString = System.getenv(Environment.NM_HOST.name()); String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV); if (clientVersion == null) { clientVersion = VersionInfo.UNKNOWN; } // TODO Should this be defaulting to 1. Was there a version of YARN where this was not setup ? int maxAppAttempts = 1; String maxAppAttemptsEnv = System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); if (maxAppAttemptsEnv != null) { maxAppAttempts = Integer.valueOf(maxAppAttemptsEnv); } validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); long appSubmitTime = Long.parseLong(appSubmitTimeStr); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); // Command line options Options opts = new Options(); opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION, false, "Run Tez Application Master in Session mode"); CommandLine cliParser = new GnuParser().parse(opts, args); // TODO Does this really need to be a YarnConfiguration ? Configuration conf = new Configuration(new YarnConfiguration()); TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()), conf); UserGroupInformation.setConfiguration(conf); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DAGAppMaster appMaster = new DAGAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime, cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION), System.getenv(Environment.PWD.name()), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())), clientVersion, maxAppAttempts, credentials, jobUserName); ShutdownHookManager.get().addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); initAndStartAppMaster(appMaster, conf); } catch (Throwable t) { LOG.fatal("Error starting DAGAppMaster", t); System.exit(1); } }
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { this(taskSchedulerContext, new SystemClock()); }
From source file:org.apache.tez.dag.app.TestDAGAppMaster.java
License:Apache License
@SuppressWarnings("deprecation") private void testDagCredentials(boolean doMerge) throws IOException { TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge); conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString()); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); // create some sample AM credentials Credentials amCreds = new Credentials(); JobTokenSecretManager jtsm = new JobTokenSecretManager(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString())); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, amCreds); TestTokenSecretManager ttsm = new TestTokenSecretManager(); Text tokenAlias1 = new Text("alias1"); Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken1")), ttsm); amCreds.addToken(tokenAlias1, amToken1); Text tokenAlias2 = new Text("alias2"); Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken2")), ttsm); amCreds.addToken(tokenAlias2, amToken2); FileSystem fs = FileSystem.getLocal(conf); FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream); sessionJarsPBOutStream.close();//from ww w . ja v a 2 s . com DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), 1, amCreds, "someuser", null); am.init(conf); am.start(); // create some sample DAG credentials Credentials dagCreds = new Credentials(); Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken1")), ttsm); dagCreds.addToken(tokenAlias2, dagToken1); Text tokenAlias3 = new Text("alias3"); Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken2")), ttsm); dagCreds.addToken(tokenAlias3, dagToken2); TezDAGID dagId = TezDAGID.getInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag") .setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build(); DAGImpl dag = am.createDAG(dagPlan, dagId); Credentials fetchedDagCreds = dag.getCredentials(); am.stop(); Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1); if (doMerge) { assertNotNull("AM creds missing from DAG creds", fetchedToken1); compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1)); } else { assertNull("AM creds leaked to DAG creds", fetchedToken1); } compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2)); compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3)); }
From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java
License:Apache License
@Test(timeout = 5000) public void testDrainEvents() throws IOException { Configuration conf = new Configuration(); AppContext appContext = mock(AppContext.class); when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR)); when(appContext.getClock()).thenReturn(new SystemClock()); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); when(appContext.getApplicationID()).thenReturn(appId); MockRecoveryService recoveryService = new MockRecoveryService(appContext); conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true); recoveryService.init(conf);/* www .j a v a 2 s. c om*/ recoveryService.start(); TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1); int randEventCount = new Random().nextInt(100) + 100; for (int i = 0; i < randEventCount; ++i) { recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent( TezTaskID.getInstance(TezVertexID.getInstance(dagId, 1), 1), "v1", 0L, 0L))); } recoveryService.stop(); assertEquals(randEventCount, recoveryService.processedRecoveryEventCounter.get()); }
From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java
License:Apache License
@Test(timeout = 5000) public void testMultipleDAGFinishedEvent() throws IOException { Configuration conf = new Configuration(); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); AppContext appContext = mock(AppContext.class); when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR)); when(appContext.getClock()).thenReturn(new SystemClock()); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); when(appContext.getApplicationID()).thenReturn(appId); MockRecoveryService recoveryService = new MockRecoveryService(appContext); conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true); recoveryService.init(conf);/*from w ww . jav a 2 s.co m*/ recoveryService.start(); TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1); int randEventCount = new Random().nextInt(100) + 100; for (int i = 0; i < randEventCount; ++i) { recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent( TezTaskID.getInstance(TezVertexID.getInstance(dagId, 1), 1), "v1", 0L, 0L))); } recoveryService.await(); assertTrue(recoveryService.outputStreamMap.containsKey(dagId)); // 2 DAGFinishedEvent recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, null))); // outputStream removed assertFalse(recoveryService.outputStreamMap.containsKey(dagId)); recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR, "diag", null, "user", "dag1", null, appAttemptId, null))); // no new outputStream opened assertEquals(recoveryService.outputStreamMap.size(), 0); assertFalse(recoveryService.outputStreamMap.containsKey(dagId)); recoveryService.stop(); }
From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java
License:Apache License
@Test(timeout = 5000) public void testSummaryPathExisted() throws IOException { Configuration conf = new Configuration(); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); AppContext appContext = mock(AppContext.class); when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR)); when(appContext.getClock()).thenReturn(new SystemClock()); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); when(appContext.getApplicationID()).thenReturn(appId); MockRecoveryService recoveryService = new MockRecoveryService(appContext); conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true); recoveryService.init(conf);/*from w w w . ja v a2 s .co m*/ recoveryService.start(); TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1); Path dagRecoveryPath = TezCommonUtils.getSummaryRecoveryPath(recoveryService.recoveryPath); touchFile(dagRecoveryPath); assertFalse(recoveryService.hasRecoveryFailed()); recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR, "diag", null, "user", "dag1", null, appAttemptId, null))); assertTrue(recoveryService.hasRecoveryFailed()); // be able to handle event after fatal error recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR, "diag", null, "user", "dag1", null, appAttemptId, null))); }