List of usage examples for org.apache.hadoop.yarn.api.records ApplicationId newInstance
@Public
@Unstable
public static ApplicationId newInstance(long clusterTimestamp, int id)
From source file:org.apache.tez.mapreduce.combine.TestMRCombiner.java
License:Apache License
private TaskContext getTaskContext(TezConfiguration conf) throws IOException { UserPayload payload = TezUtils.createUserPayloadFromConf(conf); TaskContext taskContext = Mockito.mock(InputContext.class); Mockito.when(taskContext.getUserPayload()).thenReturn(payload); Mockito.when(taskContext.getCounters()).thenReturn(new TezCounters()); Mockito.when(taskContext.getApplicationId()).thenReturn(ApplicationId.newInstance(123456, 1)); return taskContext; }
From source file:org.apache.tez.mapreduce.hadoop.IDConverter.java
License:Apache License
public static TezDAGID fromMRJobId(org.apache.hadoop.mapreduce.JobID jobId) { return TezDAGID.getInstance(ApplicationId.newInstance(Long.valueOf(jobId.getJtIdentifier()), jobId.getId()), 1);//from ww w .ja v a 2 s. co m }
From source file:org.apache.tez.mapreduce.input.TestMRInput.java
License:Apache License
@Test(timeout = 5000) public void test0PhysicalInputs() throws IOException { InputContext inputContext = mock(InputContext.class); DataSourceDescriptor dsd = MRInput/*from www .j ava2s.c o m*/ .createConfigBuilder(new Configuration(false), FileInputFormat.class, "testPath").build(); ApplicationId applicationId = ApplicationId.newInstance(1000, 1); doReturn(dsd.getInputDescriptor().getUserPayload()).when(inputContext).getUserPayload(); doReturn(applicationId).when(inputContext).getApplicationId(); doReturn("dagName").when(inputContext).getDAGName(); doReturn("vertexName").when(inputContext).getTaskVertexName(); doReturn("inputName").when(inputContext).getSourceVertexName(); doReturn("uniqueIdentifier").when(inputContext).getUniqueIdentifier(); doReturn(1).when(inputContext).getTaskIndex(); doReturn(1).when(inputContext).getTaskAttemptNumber(); doReturn(new TezCounters()).when(inputContext).getCounters(); MRInput mrInput = new MRInput(inputContext, 0); mrInput.initialize(); mrInput.start(); assertFalse(mrInput.getReader().next()); verify(inputContext, times(1)).notifyProgress(); List<Event> events = new LinkedList<>(); try { mrInput.handleEvents(events); fail("HandleEvents should cause an input with 0 physical inputs to fail"); } catch (Exception e) { assertTrue(e instanceof IllegalStateException); } }
From source file:org.apache.tez.mapreduce.input.TestMultiMRInput.java
License:Apache License
private InputContext createTezInputContext(byte[] payload) { ApplicationId applicationId = ApplicationId.newInstance(10000, 1); TezCounters counters = new TezCounters(); InputContext inputContext = mock(InputContext.class); doReturn(applicationId).when(inputContext).getApplicationId(); doReturn(counters).when(inputContext).getCounters(); doReturn(1).when(inputContext).getDAGAttemptNumber(); doReturn("dagName").when(inputContext).getDAGName(); doReturn(1).when(inputContext).getInputIndex(); doReturn("srcVertexName").when(inputContext).getSourceVertexName(); doReturn(1).when(inputContext).getTaskAttemptNumber(); doReturn(1).when(inputContext).getTaskIndex(); doReturn(1).when(inputContext).getTaskVertexIndex(); doReturn("taskVertexName").when(inputContext).getTaskVertexName(); doReturn(UserPayload.create(ByteBuffer.wrap(payload))).when(inputContext).getUserPayload(); return inputContext; }
From source file:org.apache.tez.mapreduce.output.TestMROutput.java
License:Apache License
private OutputContext createMockOutputContext(UserPayload payload) { OutputContext outputContext = mock(OutputContext.class); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); when(outputContext.getUserPayload()).thenReturn(payload); when(outputContext.getApplicationId()).thenReturn(appId); when(outputContext.getTaskVertexIndex()).thenReturn(1); when(outputContext.getTaskAttemptNumber()).thenReturn(1); when(outputContext.getCounters()).thenReturn(new TezCounters()); return outputContext; }
From source file:org.apache.tez.mapreduce.output.TestMultiMROutput.java
License:Apache License
private OutputContext createMockOutputContext(UserPayload payload) { OutputContext outputContext = mock(OutputContext.class); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); when(outputContext.getUserPayload()).thenReturn(payload); when(outputContext.getApplicationId()).thenReturn(appId); when(outputContext.getTaskVertexIndex()).thenReturn(1); when(outputContext.getTaskAttemptNumber()).thenReturn(1); when(outputContext.getCounters()).thenReturn(new TezCounters()); when(outputContext.getStatisticsReporter()).thenReturn(mock(OutputStatisticsReporter.class)); return outputContext; }
From source file:org.apache.tez.mapreduce.TezTestUtils.java
License:Apache License
public static TezTaskAttemptID getMockTaskAttemptId(int jobId, int vertexId, int taskId, int taskAttemptId) { return TezTaskAttemptID.getInstance( TezTaskID.getInstance(TezVertexID.getInstance( TezDAGID.getInstance(ApplicationId.newInstance(0, jobId), jobId), vertexId), taskId), taskAttemptId);//from ww w . j av a 2 s.co m }
From source file:org.apache.tez.mapreduce.TezTestUtils.java
License:Apache License
public static TezTaskID getMockTaskId(int jobId, int vertexId, int taskId) { return TezTaskID.getInstance( TezVertexID.getInstance(TezDAGID.getInstance(ApplicationId.newInstance(0, jobId), jobId), vertexId), taskId);/* w w w . ja v a 2 s.c o m*/ }
From source file:org.apache.tez.records.TezContainerId.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { long timestamp = in.readLong(); int appId = in.readInt(); int appAttemptId = in.readInt(); int id = in.readInt(); ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, appAttemptId); this.containerId = ContainerId.newInstance(applicationAttemptId, id); }
From source file:org.apache.tez.runtime.api.impl.TestProcessorContext.java
License:Apache License
@Test(timeout = 5000) public void testDagNumber() throws IOException { String[] localDirs = new String[] { "dummyLocalDir" }; int appAttemptNumber = 1; TezUmbilical tezUmbilical = mock(TezUmbilical.class); String dagName = "DAG_NAME"; String vertexName = "VERTEX_NAME"; int vertexParallelism = 20; int dagNumber = 52; ApplicationId appId = ApplicationId.newInstance(10000, 13); TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber); TezVertexID vertexId = TezVertexID.getInstance(dagId, 6); TezTaskID taskId = TezTaskID.getInstance(vertexId, 4); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2); TaskSpec mockSpec = mock(TaskSpec.class); when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class))); when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class))); LogicalIOProcessorRuntimeTask runtimeTask = new LogicalIOProcessorRuntimeTask(mockSpec, 1, new Configuration(), new String[] { "/" }, tezUmbilical, null, null, null, null, "", null, 1024, false, new DefaultHadoopShim()); LogicalIOProcessorRuntimeTask mockTask = spy(runtimeTask); Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap(); Map<String, String> auxServiceEnv = Maps.newHashMap(); MemoryDistributor memDist = mock(MemoryDistributor.class); ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class); InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class); ObjectRegistry objectRegistry = new ObjectRegistryImpl(); ExecutionContext execContext = new ExecutionContextImpl("localhost"); long memAvailable = 10000l; TezProcessorContextImpl procContext = new TezProcessorContextImpl(new Configuration(), localDirs, appAttemptNumber, tezUmbilical, dagName, vertexName, vertexParallelism, taskAttemptId, null, runtimeTask, serviceConsumerMetadata, auxServiceEnv, memDist, processorDesc, inputReadyTracker, objectRegistry, execContext, memAvailable); assertEquals(dagNumber, procContext.getDagIdentifier()); assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber()); assertEquals(appId, procContext.getApplicationId()); assertEquals(dagName, procContext.getDAGName()); assertEquals(vertexName, procContext.getTaskVertexName()); assertEquals(vertexId.getId(), procContext.getTaskVertexIndex()); assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs())); // test auto call of notifyProgress procContext.setProgress(0.1f);/* w w w . j a v a2 s . c om*/ verify(mockTask, times(1)).notifyProgressInvocation(); }