List of usage examples for org.apache.hadoop.yarn.api.records LocalResource newInstance
@Public
@Stable
public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility,
long size, long timestamp)
From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java
License:Apache License
@Override public ContainerLaunchContext getLaunchContext(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {//w ww . j ava2 s. c om Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } FileSystem fs = FileSystem.get(appContext.getConfiguration()); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String suffix = "Tajo" + "/" + appContext.getApplicationId().getId(); Path parentPath = new Path(fs.getHomeDirectory(), suffix); // tar ball Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH)); FileStatus archiveFs = fs.getFileStatus(archivePath); LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(), archiveFs.getModificationTime()); localResources.put("tajo", archiveRsrc); Configuration tajoWorkerConf = new Configuration(false); tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml")); tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001"); tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005"); Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf"); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoWorkerConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); launchContext.setLocalResources(localResources); // Set the environment setupEnv(launchContext); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); // Set executable command // Set args for the shell command if any vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo"); vargs.add("--config"); vargs.add("${" + Constants.TAJO_CONF_DIR + "}"); vargs.add("worker"); // Add log redirect params // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); launchContext.setCommands(commands); return launchContext; }
From source file:org.apache.tez.client.TestTezClient.java
License:Apache License
public void testTezClient(boolean isSession) throws Exception { Map<String, LocalResource> lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); TezClientForTest client = configure(lrs, isSession); ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor .forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client.start();//from w w w .ja v a 2s .c om verify(client.mockYarnClient, times(1)).init((Configuration) any()); verify(client.mockYarnClient, times(1)).start(); if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(3, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); } else { verify(client.mockYarnClient, times(0)).submitApplication(captor.capture()); } String mockLR1Name = "LR1"; Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)); DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); DAGClient dagClient = client.submitDAG(dag); Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); if (isSession) { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); verify(client.sessionAmProxy, times(1)).submitDAG((RpcController) any(), (SubmitDAGRequestProto) any()); } else { verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(4, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); } // add resources String lrName2 = "LR2"; lrs.clear(); lrs.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); client.addAppMasterLocalFiles(lrs); ApplicationId appId2 = ApplicationId.newInstance(0, 2); when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId()) .thenReturn(appId2); when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); dag = DAG.create("DAG") .addVertex(Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1))); dagClient = client.submitDAG(dag); if (isSession) { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); // additional resource is sent ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(2)).submitDAG((RpcController) any(), captor1.capture()); SubmitDAGRequestProto proto = captor1.getValue(); Assert.assertEquals(1, proto.getAdditionalAmResources().getLocalResourcesCount()); Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName()); } else { // new app master Assert.assertTrue(dagClient.getExecutionContext().contains(appId2.toString())); verify(client.mockYarnClient, times(2)).submitApplication(captor.capture()); // additional resource is added ApplicationSubmissionContext context = captor.getValue(); Assert.assertEquals(5, context.getAMContainerSpec().getLocalResources().size()); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources() .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1)); Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName2)); } client.stop(); if (isSession) { verify(client.sessionAmProxy, times(1)).shutdownSession((RpcController) any(), (ShutdownSessionRequestProto) any()); } verify(client.mockYarnClient, times(1)).stop(); }
From source file:org.apache.tez.client.TestTezClient.java
License:Apache License
public void testMultipleSubmissionsJob(boolean isSession) throws Exception { TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession); when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client1.start();//from ww w. j a va 2 s .co m String mockLR1Name = "LR1"; Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); String mockLR2Name = "LR2"; Map<String, LocalResource> lrVertex = Collections.singletonMap(mockLR2Name, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)) .addTaskLocalFiles(lrVertex); DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); // the dag resource will be added to the vertex once client1.submitDAG(dag); TezClientForTest client2 = configure(); when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client2.start(); // verify resubmission of same dag to new client (simulates submission error resulting in the // creation of a new client and resubmission of the DAG) client2.submitDAG(dag); client1.stop(); client2.stop(); }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Setup LocalResource map for Tez jars based on provided Configuration * //ww w . j ava 2 s . c o m * @param conf * Configuration to use to access Tez jars' locations * @param credentials * a credentials instance into which tokens for the Tez local * resources will be populated * @param tezJarResources Map of LocalResources to use for AM and DAGs * @return Whether the archive-based deployment of Tez was used. * @throws IOException */ static boolean setupTezJarsLocalResources(TezConfiguration conf, Credentials credentials, Map<String, LocalResource> tezJarResources) throws IOException { Preconditions.checkNotNull(credentials, "A non-null credentials object should be specified"); boolean usingTezArchive = false; if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) { LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since '" + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true"); } else { // Add tez jars to local resource String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS); if (tezJarUris == null || tezJarUris.length == 0) { throw new TezUncheckedException("Invalid configuration of tez jars" + ", " + TezConfiguration.TEZ_LIB_URIS + " is not defined in the configuration"); } LOG.info("Using tez.lib.uris value from configuration: " + conf.get(TezConfiguration.TEZ_LIB_URIS)); if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz") || tezJarUris[0].endsWith(".zip") || tezJarUris[0].endsWith(".tar"))) { String fileName = tezJarUris[0]; FileStatus fStatus = getLRFileStatus(fileName, conf)[0]; LocalResourceVisibility lrVisibility; if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE) && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; } else { lrVisibility = LocalResourceVisibility.PRIVATE; } tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME, LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()), LocalResourceType.ARCHIVE, lrVisibility, fStatus.getLen(), fStatus.getModificationTime())); Path[] tezJarPaths = { fStatus.getPath() }; // obtain credentials TokenCache.obtainTokensForFileSystems(credentials, tezJarPaths, conf); usingTezArchive = true; } else { // Treat as non-archives addLocalResources(conf, tezJarUris, tezJarResources, credentials); } if (tezJarResources.isEmpty()) { throw new TezUncheckedException("No files found in locations specified in " + TezConfiguration.TEZ_LIB_URIS + " . Locations: " + StringUtils.join(tezJarUris, ',')); } } // Add aux uris to local resources addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS), tezJarResources, credentials); return usingTezArchive; }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
private static void addLocalResources(Configuration conf, String[] configUris, Map<String, LocalResource> tezJarResources, Credentials credentials) throws IOException { if (configUris == null || configUris.length == 0) { return;/* w w w. j a v a 2s .c o m*/ } List<Path> configuredPaths = Lists.newArrayListWithCapacity(configUris.length); for (String configUri : configUris) { boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, configUri, FsAction.EXECUTE); FileStatus[] fileStatuses = getLRFileStatus(configUri, conf); for (FileStatus fStatus : fileStatuses) { if (fStatus.isDirectory()) { // Skip directories - no recursive search support. continue; } LocalResourceVisibility lrVisibility; if (ancestorsHavePermission && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; } else { lrVisibility = LocalResourceVisibility.PRIVATE; } String rsrcName = fStatus.getPath().getName(); if (tezJarResources.containsKey(rsrcName)) { String message = "Duplicate resource found" + ", resourceName=" + rsrcName + ", existingPath=" + tezJarResources.get(rsrcName).getResource().toString() + ", newPath=" + fStatus.getPath(); LOG.warn(message); } tezJarResources.put(rsrcName, LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()), LocalResourceType.FILE, lrVisibility, fStatus.getLen(), fStatus.getModificationTime())); configuredPaths.add(fStatus.getPath()); } } // Obtain credentials. if (!configuredPaths.isEmpty()) { TokenCache.obtainTokensForFileSystems(credentials, configuredPaths.toArray(new Path[configuredPaths.size()]), conf); } }
From source file:org.apache.tez.common.TestTezCommonUtils.java
License:Apache License
@Test(timeout = 5000) public void testAddAdditionalLocalResources() { String lrName = "LR"; Map<String, LocalResource> originalLrs; originalLrs = Maps.newHashMap();//w ww. j a v a2 s . c o m originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> additionalLrs; // Same path, same size. originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Same path, different size. originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); try { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); Assert.fail("Duplicate LRs with different sizes expected to fail"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } // Different path, same size, diff timestamp originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 100)); TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Different path, different size originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); additionalLrs = Maps.newHashMap(); additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); try { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); Assert.fail("Duplicate LRs with different sizes expected to fail"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } }
From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java
License:Apache License
@Test(timeout = 5000) @SuppressWarnings("unchecked") public void testSubmitDagInSessionWithLargeDagPlan() throws Exception { int maxIPCMsgSize = 1024; String dagPlanName = "dagplan-name"; File requestFile = tmpFolder.newFile("request-file"); TezConfiguration conf = new TezConfiguration(); conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize); byte[] randomBytes = new byte[2 * maxIPCMsgSize]; (new Random()).nextBytes(randomBytes); UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes)); Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1); DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false); String lrName = "localResource"; String scheme = "file"; String host = "localhost"; int port = 80; String path = "/test"; URL lrURL = URL.newInstance(scheme, host, port, path); LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1); Map<String, LocalResource> localResources = new HashMap<>(); localResources.put(lrName, localResource); SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan) .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources)); try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) { requestBuilder.build().writeTo(fileOutputStream); }/*from w ww . j a v a2 s. c om*/ DAGClientHandler dagClientHandler = mock(DAGClientHandler.class); ACLManager aclManager = mock(ACLManager.class); DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy( new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf))); when(dagClientHandler.getACLManager()).thenReturn(aclManager); when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id"); when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true); requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath()); serverImpl.submitDAG(null, requestBuilder.build()); ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class); verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture()); dagPlan = dagPlanCaptor.getValue(); localResources = localResourcesCaptor.getValue(); assertEquals(dagPlan.getName(), dagPlanName); assertEquals(dagPlan.getVertexCount(), 1); assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize); assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()); assertEquals(localResources.size(), 1); assertTrue(localResources.containsKey(lrName)); localResource = localResources.get(lrName); assertEquals(localResource.getType(), LocalResourceType.FILE); assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC); lrURL = localResource.getResource(); assertEquals(lrURL.getScheme(), scheme); assertEquals(lrURL.getHost(), host); assertEquals(lrURL.getPort(), port); assertEquals(lrURL.getFile(), path); }
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testDAGCreateDataInference() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName)); Map<String, LocalResource> lrs1 = Maps.newHashMap(); String lrName1 = "LR1"; lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrs2 = Maps.newHashMap(); String lrName2 = "LR2"; lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Set<String> hosts = Sets.newHashSet(); hosts.add("h1"); hosts.add("h2"); List<TaskLocationHint> taskLocationHints = Lists.newLinkedList(); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints); DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2); v1.addDataSource("i1", ds); DAG dag = DAG.create("testDag"); dag.addVertex(v1);/* w w w. j a v a 2 s . com*/ dag.addTaskLocalFiles(lrs1); DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true); Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName()); VertexPlan vPlan = dagPlan.getVertex(0); PlanTaskConfiguration taskPlan = vPlan.getTaskConfig(); Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks()); Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb()); Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName()); Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount()); }
From source file:org.apache.tez.dag.api.TestDAGVerify.java
License:Apache License
@Test(timeout = 5000) public void testInferredFilesFail() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName), dummyTaskCount, dummyTaskResource);/*from w w w . j ava 2 s .co m*/ Map<String, LocalResource> lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); // Same lr, different size Map<String, LocalResource> lrs2 = Maps.newHashMap(); lrs2.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1)); v1.addTaskLocalFiles(lrs); // Allowed since the LR is the same. try { v1.addTaskLocalFiles(lrs2); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), null, -1, null, null, lrs2); v1.addDataSource("i1", ds); DAG dag = DAG.create("testDag"); dag.addVertex(v1); dag.addTaskLocalFiles(lrs); try { dag.addTaskLocalFiles(lrs2); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } try { // data source will add duplicate common files to vertex dag.createDag(new TezConfiguration(), null, null, null, true); Assert.fail(); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } }
From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java
License:Apache License
/** * Create a {@link LocalResource} record with all the given parameters. *///from www . jav a 2 s . c om public static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat.getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }