List of usage examples for org.apache.hadoop.yarn.api.records URL newInstance
@Public
@Stable
public static URL newInstance(String scheme, String host, int port, String file)
From source file:org.apache.tez.client.TestTezClient.java
License:Apache License
public void testTezClient(boolean isSession) throws Exception { Map<String, LocalResource> lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); TezClientForTest client = configure(lrs, isSession); ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor .forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client.start();/*w ww.jav a 2 s. co m*/ verify(client.mockYarnClient, times(1)).init((Configuration) any()); verify(client.mockYarnClient, times(1)).start(); if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(3, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); } else { verify(client.mockYarnClient, times(0)).submitApplication(captor.capture()); } String mockLR1Name = "LR1"; Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)); DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); DAGClient dagClient = client.submitDAG(dag); Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); verify(client.sessionAmProxy, times(1)).submitDAG((RpcController) any(), (SubmitDAGRequestProto) any()); } else { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(4, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); } // add resources String lrName2 = "LR2"; lrs.clear(); lrs.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); client.addAppMasterLocalFiles(lrs); ApplicationId appId2 = ApplicationId.newInstance(0, 2); when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId()) .thenReturn(appId2); when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); dag = DAG.create("DAG") .addVertex(Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1))); dagClient = client.submitDAG(dag); if (isSession) { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); // additional resource is sent ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(2)).submitDAG((RpcController) any(), captor1.capture()); SubmitDAGRequestProto proto = captor1.getValue(); Assert.assertEquals(1, proto.getAdditionalAmResources().getLocalResourcesCount()); Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName()); } else { // new app master Assert.assertTrue(dagClient.getExecutionContext().contains(appId2.toString())); verify(client.mockYarnClient, times(2)).submitApplication(captor.capture()); // additional resource is added ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(5, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName2)); } client.stop(); if (isSession) { verify(client.sessionAmProxy, times(1)).shutdownSession((RpcController) any(), (ShutdownSessionRequestProto) any()); } verify(client.mockYarnClient, times(1)).stop(); }
From source file:org.apache.tez.client.TestTezClient.java
License:Apache License
public void testMultipleSubmissionsJob(boolean isSession) throws Exception { TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession); when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client1.start();/*from w w w . j a v a2 s . c o m*/ String mockLR1Name = "LR1"; Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); String mockLR2Name = "LR2"; Map<String, LocalResource> lrVertex = Collections.singletonMap(mockLR2Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)) .addTaskLocalFiles(lrVertex); DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); // the dag resource will be added to the vertex once client1.submitDAG(dag); TezClientForTest client2 = configure(); when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client2.start(); // verify resubmission of same dag to new client (simulates submission error resulting in the // creation of a new client and resubmission of the DAG) client2.submitDAG(dag); client1.stop(); client2.stop(); }
From source file:org.apache.tez.common.TestTezCommonUtils.java
License:Apache License
@Test(timeout = 5000) public void testAddAdditionalLocalResources() { String lrName = "LR"; Map<String, LocalResource> originalLrs; originalLrs = Maps.newHashMap();//w w w . j av a 2s. c o m originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> additionalLrs; // Same path, same size. originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Same path, different size. originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); try { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); Assert.fail("Duplicate LRs with different sizes expected to fail"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } // Different path, same size, diff timestamp originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 100)); TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Different path, different size originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); try { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); Assert.fail("Duplicate LRs with different sizes expected to fail"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } }
From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java
License:Apache License
@Test(timeout = 5000) @SuppressWarnings("unchecked") public void testSubmitDagInSessionWithLargeDagPlan() throws Exception { int maxIPCMsgSize = 1024; String dagPlanName = "dagplan-name"; File requestFile = tmpFolder.newFile("request-file"); TezConfiguration conf = new TezConfiguration(); conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize); byte[] randomBytes = new byte[2 * maxIPCMsgSize]; (new Random()).nextBytes(randomBytes); UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes)); Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1); DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false); String lrName = "localResource"; String scheme = "file"; String host = "localhost"; int port = 80; String path = "/test"; URL lrURL = URL.newInstance(scheme, host, port, path); LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1); Map<String, LocalResource> localResources = new HashMap<>(); localResources.put(lrName, localResource); SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan) .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources)); try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) { requestBuilder.build().writeTo(fileOutputStream); }/* w w w. j ava2 s . c o m*/ DAGClientHandler dagClientHandler = mock(DAGClientHandler.class); ACLManager aclManager = mock(ACLManager.class); DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy( new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf))); when(dagClientHandler.getACLManager()).thenReturn(aclManager); when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id"); when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true); requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath()); serverImpl.submitDAG(null, requestBuilder.build()); ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class); verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture()); dagPlan = dagPlanCaptor.getValue(); localResources = localResourcesCaptor.getValue(); assertEquals(dagPlan.getName(), dagPlanName); assertEquals(dagPlan.getVertexCount(), 1); assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize); assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()); assertEquals(localResources.size(), 1); assertTrue(localResources.containsKey(lrName)); localResource = localResources.get(lrName); assertEquals(localResource.getType(), LocalResourceType.FILE); assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC); lrURL = localResource.getResource(); assertEquals(lrURL.getScheme(), scheme); assertEquals(lrURL.getHost(), host); assertEquals(lrURL.getPort(), port); assertEquals(lrURL.getFile(), path); }
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testDAGCreateDataInference() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName)); Map<String, LocalResource> lrs1 = Maps.newHashMap(); String lrName1 = "LR1"; lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrs2 = Maps.newHashMap(); String lrName2 = "LR2"; lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Set<String> hosts = Sets.newHashSet(); hosts.add("h1"); hosts.add("h2"); List<TaskLocationHint> taskLocationHints = Lists.newLinkedList(); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints); DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2); v1.addDataSource("i1", ds); DAG dag = DAG.create("testDag"); dag.addVertex(v1);// ww w.j a v a 2s. co m dag.addTaskLocalFiles(lrs1); DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true); Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName()); VertexPlan vPlan = dagPlan.getVertex(0); PlanTaskConfiguration taskPlan = vPlan.getTaskConfig(); Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks()); Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb()); Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName()); Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount()); }
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testInferredFilesFail() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName), dummyTaskCount, dummyTaskResource);//from w w w .ja v a2s .c o m Map<String, LocalResource> lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); // Same lr, different size Map<String, LocalResource> lrs2 = Maps.newHashMap(); lrs2.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); v1.addTaskLocalFiles(lrs); // Allowed since the LR is the same. try { v1.addTaskLocalFiles(lrs2); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), null, -1, null, null, lrs2); v1.addDataSource("i1", ds); DAG dag = DAG.create("testDag"); dag.addVertex(v1); dag.addTaskLocalFiles(lrs); try { dag.addTaskLocalFiles(lrs2); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } try { // data source will add duplicate common files to vertex dag.createDag(new TezConfiguration(), null, null, null, true); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } }
From source file:org.apache.tez.dag.app.rm.container.TestAMContainer.java
License:Apache License
private LocalResource createLocalResource(String name) { LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000); return lr;//w w w. j a va2s . c om }
From source file:org.apache.tez.dag.app.TestMockDAGAppMaster.java
License:Apache License
@Test(timeout = 5000) public void testLocalResourceSetup() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start();/*from ww w .j ava2s .c o m*/ MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); Map<String, LocalResource> lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrVertex = Maps.newHashMap(); String lrName2 = "LR2"; lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex); dag.addVertex(vA); DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); ContainerData cData = mockLauncher.getContainers().values().iterator().next(); ContainerLaunchContext launchContext = cData.launchContext; Map<String, LocalResource> taskLR = launchContext.getLocalResources(); // verify tasks are launched with both DAG and task resources. Assert.assertTrue(taskLR.containsKey(lrName1)); Assert.assertTrue(taskLR.containsKey(lrName2)); mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); }
From source file:org.apache.tez.dag.app.TestMockDAGAppMaster.java
License:Apache License
@Test(timeout = 10000) public void testMultipleSubmissions() throws Exception { Map<String, LocalResource> lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrVertex = Maps.newHashMap(); String lrName2 = "LR2"; lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex); dag.addVertex(vA);/*from ww w.j ava2 s . c o m*/ TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); DAGClient dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); // submit the same DAG again to verify it can be done. tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); }