List of usage examples for org.apache.hadoop.yarn.api.records Resource newInstance
@Public
@Stable
public static Resource newInstance(long memory, int vCores)
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testMultipleRootInputsAllowed() { DAG dag = DAG.create("testDag"); ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1") .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes()))); Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1)); VertexManagerPluginDescriptor vertexManagerPluginDescriptor = VertexManagerPluginDescriptor .create("TestVertexManager"); v1.setVertexManagerPlugin(vertexManagerPluginDescriptor); InputDescriptor inputDescriptor1 = InputDescriptor.create("input1") .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes()))); InputDescriptor inputDescriptor2 = InputDescriptor.create("input2") .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes()))); v1.addDataSource("input1", DataSourceDescriptor.create(inputDescriptor1, null, null)); v1.addDataSource("input2", DataSourceDescriptor.create(inputDescriptor2, null, null)); dag.addVertex(v1);//from w w w. java 2 s . c o m dag.createDag(new TezConfiguration(), null, null, null, true); }
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testDAGAccessControls() { DAG dag = DAG.create("testDag"); ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1") .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes()))); Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1)); dag.addVertex(v1);//from ww w .j a v a2s .c o m DAGAccessControls dagAccessControls = new DAGAccessControls(); dagAccessControls.setUsersWithViewACLs(Arrays.asList("u1")).setUsersWithModifyACLs(Arrays.asList("*")) .setGroupsWithViewACLs(Arrays.asList("g1")).setGroupsWithModifyACLs(Arrays.asList("g2")); dag.setAccessControls(dagAccessControls); Configuration conf = new Configuration(false); DAGPlan dagPlan = dag.createDag(conf, null, null, null, true); Assert.assertNull(conf.get(TezConstants.TEZ_DAG_VIEW_ACLS)); Assert.assertNull(conf.get(TezConstants.TEZ_DAG_MODIFY_ACLS)); ConfigurationProto confProto = dagPlan.getDagConf(); boolean foundViewAcls = false; boolean foundModifyAcls = false; for (PlanKeyValuePair pair : confProto.getConfKeyValuesList()) { if (pair.getKey().equals(TezConstants.TEZ_DAG_VIEW_ACLS)) { foundViewAcls = true; Assert.assertEquals("u1 g1", pair.getValue()); } else if (pair.getKey().equals(TezConstants.TEZ_DAG_MODIFY_ACLS)) { foundModifyAcls = true; Assert.assertEquals("*", pair.getValue()); } } Assert.assertTrue(foundViewAcls); Assert.assertTrue(foundModifyAcls); }
From source file:org.apache.tez.dag.app.dag.impl.DAGSchedulerMRR.java
License:Apache License
@Override public void scheduleTask(DAGEventSchedulerUpdate event) { TaskAttempt attempt = event.getAttempt(); Vertex vertex = dag.getVertex(attempt.getVertexID()); int vertexDistanceFromRoot = vertex.getDistanceFromRoot(); LOG.info("Schedule task: " + attempt.getID()); if (currentPartitioner == null) { // no partitioner. so set it. currentPartitioner = vertex;/* w w w. ja v a 2s . co m*/ currentShufflerDepth = vertexDistanceFromRoot; assert realPartitionerResource == null; Resource partitionerResource = currentPartitioner.getTaskResource(); realPartitionerResource = Resource.newInstance(partitionerResource.getMemory(), partitionerResource.getVirtualCores()); LOG.info(vertex.getVertexId() + " is new partitioner at depth " + vertexDistanceFromRoot); } else if (currentShuffler == null && vertexDistanceFromRoot > currentShufflerDepth) { // vertex not a partitioner. no shuffler set. has more depth than current // shuffler. this must be the new shuffler. currentShuffler = vertex; currentShufflerDepth = vertexDistanceFromRoot; assert realShufflerResource == null; Resource shufflerResource = currentShuffler.getTaskResource(); realShufflerResource = Resource.newInstance(shufflerResource.getMemory(), shufflerResource.getVirtualCores()); LOG.info(vertex.getVertexId() + " is new shuffler at depth " + currentShufflerDepth); } if (currentShuffler == vertex) { pendingShuffleTasks.add(attempt); unassignedShuffleTasks.add(attempt.getTaskID()); schedulePendingShuffles(getNumShufflesToSchedule()); return; } if (currentPartitioner == vertex) { unassignedPartitionTasks.add(attempt.getTaskID()); } // sanity check // task should be a partitioner, a shuffler or a retry of an ancestor if (currentPartitioner != vertex && currentShuffler != vertex && vertexDistanceFromRoot >= currentPartitioner.getDistanceFromRoot()) { String message = vertex.getVertexId() + " is neither the " + " current partitioner: " + currentPartitioner.getVertexId() + " nor the current shuffler: " + currentShuffler.getVertexId(); LOG.fatal(message); throw new TezUncheckedException(message); } scheduleTaskAttempt(attempt); }
From source file:org.apache.tez.dag.app.dag.impl.TestCommit.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupDAG(DAGPlan dagPlan) {
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1);
dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
Assert.assertNotNull(dagId);//from w w w .jav a 2 s . c om
dispatcher = new DrainDispatcher();
fsTokens = new Credentials();
appContext = mock(AppContext.class);
when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
rawExecutor = Executors.newCachedThreadPool(
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("App Shared Pool - " + "#%d").build());
execService = MoreExecutors.listeningDecorator(rawExecutor);
doReturn(execService).when(appContext).getExecService();
historyEventHandler = new MockHistoryEventHandler(appContext);
aclManager = new ACLManager("amUser");
doReturn(conf).when(appContext).getAMConf();
doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
doReturn(dagId).when(appContext).getCurrentDAGID();
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(aclManager).when(appContext).getAMACLManager();
dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
fsTokens, clock, "user", thh, appContext);
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192, 10));
doReturn(clusterInfo).when(appContext).getClusterInfo();
dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
taskEventDispatcher = new TaskEventDispatcher();
dispatcher.register(TaskEventType.class, taskEventDispatcher);
taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
vertexEventDispatcher = new VertexEventDispatcher();
dispatcher.register(VertexEventType.class, vertexEventDispatcher);
dagEventDispatcher = new DagEventDispatcher();
dispatcher.register(DAGEventType.class, dagEventDispatcher);
dagFinishEventHandler = new DAGFinishEventHandler();
dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
dispatcher.init(conf);
dispatcher.start();
}
From source file:org.apache.tez.dag.app.dag.impl.TestCommit.java
License:Apache License
private DAGPlan createDAGPlan(boolean vertexGroupCommitSucceeded, boolean v3CommitSucceeded) throws Exception { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); DAG dag = DAG.create("testDag"); String groupName1 = "uv12"; OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap( new CountingOutputCommitter.CountingOutputCommitterConfig(!vertexGroupCommitSucceeded, true) .toUserPayload()))); OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!v3CommitSucceeded, true) .toUserPayload()))); org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2); OutputDescriptor outDesc = OutputDescriptor.create("output.class"); uv12.addDataSink("v12Out", DataSinkDescriptor.create(outDesc, ocd1, null)); v3.addDataSink("v3Out", DataSinkDescriptor.create(outDesc, ocd2, null)); GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("dummy output class"), InputDescriptor.create("dummy input class")), InputDescriptor.create("merge.class")); dag.addVertex(v1);/*from w w w. j av a2 s .c o m*/ dag.addVertex(v2); dag.addVertex(v3); dag.addEdge(e1); return dag.createDag(conf, null, null, null, true); }
From source file:org.apache.tez.dag.app.dag.impl.TestCommit.java
License:Apache License
private DAGPlan createDAGPlanWith2VertexGroupOutputs(boolean vertexGroupCommitSucceeded1, boolean vertexGroupCommitSucceeded2, boolean v3CommitSucceeded) throws Exception { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); DAG dag = DAG.create("testDag"); String groupName1 = "uv12"; OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap( new CountingOutputCommitter.CountingOutputCommitterConfig(!vertexGroupCommitSucceeded1, true).toUserPayload()))); OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap( new CountingOutputCommitter.CountingOutputCommitterConfig(!vertexGroupCommitSucceeded2, true).toUserPayload()))); OutputCommitterDescriptor ocd3 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!v3CommitSucceeded, true) .toUserPayload()))); org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2); OutputDescriptor outDesc = OutputDescriptor.create("output.class"); uv12.addDataSink("v12Out1", DataSinkDescriptor.create(outDesc, ocd1, null)); uv12.addDataSink("v12Out2", DataSinkDescriptor.create(outDesc, ocd2, null)); v3.addDataSink("v3Out", DataSinkDescriptor.create(outDesc, ocd3, null)); GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("dummy output class"), InputDescriptor.create("dummy input class")), InputDescriptor.create("merge.class")); dag.addVertex(v1);//from w w w . j a v a2s . c o m dag.addVertex(v2); dag.addVertex(v3); dag.addEdge(e1); return dag.createDag(conf, null, null, null, true); }
From source file:org.apache.tez.dag.app.dag.impl.TestCommit.java
License:Apache License
private DAGPlan createDAGPlan_SingleVertexWith2Committer(boolean commit1Succeed, boolean commit2Succeed, boolean customVM) throws IOException { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); if (customVM) { v1.setVertexManagerPlugin(/* ww w. java 2 s .c o m*/ VertexManagerPluginDescriptor.create(FailOnVMEventReceivedlVertexManager.class.getName())); } OutputCommitterDescriptor ocd1 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!commit1Succeed, true) .toUserPayload()))); OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create(CountingOutputCommitter.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig(!commit2Succeed, true) .toUserPayload()))); DAG dag = DAG.create("testDag"); dag.addVertex(v1); OutputDescriptor outDesc = OutputDescriptor.create("output.class"); v1.addDataSink("v1Out_1", DataSinkDescriptor.create(outDesc, ocd1, null)); v1.addDataSink("v1Out_2", DataSinkDescriptor.create(outDesc, ocd2, null)); return dag.createDag(conf, null, null, null, true); }
From source file:org.apache.tez.dag.app.dag.impl.TestDAGImpl.java
License:Apache License
static DAGPlan createGroupDAGPlan() { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); DAG dag = DAG.create("testDag"); String groupName1 = "uv12"; OutputCommitterDescriptor ocd = OutputCommitterDescriptor .create(TotalCountingOutputCommitter.class.getName()); org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2); OutputDescriptor outDesc = OutputDescriptor.create("output.class"); uv12.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null)); v3.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null)); GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("dummy output class"), InputDescriptor.create("dummy input class")), InputDescriptor.create("merge.class")); dag.addVertex(v1);//from w ww . ja v a 2s .c om dag.addVertex(v2); dag.addVertex(v3); dag.addEdge(e1); return dag.createDag(conf, null, null, null, true); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) public void testLocalityRequest() { TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); Set<String> hosts = new TreeSet<String>(); hosts.add("host1"); hosts.add("host2"); hosts.add("host3"); TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(hosts, null); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), locationHint, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false); TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class); sta.transition(taImpl, sEvent);/*from ww w . jav a2s. co m*/ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(1)).handle(arg.capture()); if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) { fail("Second event not of type " + AMSchedulerEventTALaunchRequest.class.getName()); } // TODO Move the Rack request check to the client after TEZ-125 is fixed. Set<String> requestedRacks = taImpl.taskRacks; assertEquals(1, requestedRacks.size()); assertEquals(3, taImpl.taskHosts.size()); for (int i = 0; i < 3; i++) { String host = ("host" + (i + 1)); assertEquals(host, true, taImpl.taskHosts.contains(host)); } }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) public void testPriority() { TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), null, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false); TaskAttemptImpl taImplReScheduled = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), null, true, Resource.newInstance(1024, 1), createFakeContainerContext(), false); ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class); when(sEvent.getPriorityLowLimit()).thenReturn(3); when(sEvent.getPriorityHighLimit()).thenReturn(1); sta.transition(taImpl, sEvent);/* w w w .java 2s . co m*/ verify(eventHandler, times(1)).handle(arg.capture()); AMSchedulerEventTALaunchRequest launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(2, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(2)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(1, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(6); when(sEvent.getPriorityHighLimit()).thenReturn(4); sta.transition(taImpl, sEvent); verify(eventHandler, times(3)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(4)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(4, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(5); when(sEvent.getPriorityHighLimit()).thenReturn(5); sta.transition(taImpl, sEvent); verify(eventHandler, times(5)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(6)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); }