Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceVisibility
  6. PUBLIC

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC.

Prototype

LocalResourceVisibility PUBLIC

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC.

Click Source Link

Document

Shared by all users on the node.

Usage

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapSuccess() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");

    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");

    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);/*w ww.  j  a  v a2  s .  co  m*/
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());

    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());

    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());

    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());

    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either",
            resourceMap.get("notExisting"));
}

From source file:org.apache.sysml.yarn.DMLYarnClient.java

License:Apache License

private Map<String, LocalResource> constructLocalResourceMap(YarnConfiguration yconf) throws IOException {
    Map<String, LocalResource> rMap = new HashMap<>();
    Path path = new Path(_hdfsJarFile);

    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = IOUtilFunctions.getFileSystem(path, yconf).getFileStatus(path);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(path));
    resource.setSize(jarStat.getLen());/*from   ww w. j av a 2s .c o m*/
    resource.setTimestamp(jarStat.getModificationTime());
    resource.setType(LocalResourceType.FILE);
    resource.setVisibility(LocalResourceVisibility.PUBLIC);

    rMap.put(DML_JAR_NAME, resource);
    return rMap;
}

From source file:org.apache.tez.client.TestTezClient.java

License:Apache License

public void testTezClient(boolean isSession) throws Exception {
    Map<String, LocalResource> lrs = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    TezClientForTest client = configure(lrs, isSession);

    ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor
            .forClass(ApplicationSubmissionContext.class);
    when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
            .thenReturn(YarnApplicationState.RUNNING);
    client.start();//from   w  w w  .  j ava  2  s. co m
    verify(client.mockYarnClient, times(1)).init((Configuration) any());
    verify(client.mockYarnClient, times(1)).start();
    if (isSession) {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(3, context.getAMContainerSpec().getLocalResources().size());
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
    } else {
        verify(client.mockYarnClient, times(0)).submitApplication(captor.capture());
    }

    String mockLR1Name = "LR1";
    Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name,
            LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE,
                    LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);
    DAGClient dagClient = client.submitDAG(dag);

    Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));

    if (isSession) {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        verify(client.sessionAmProxy, times(1)).submitDAG((RpcController) any(), (SubmitDAGRequestProto) any());
    } else {
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(4, context.getAMContainerSpec().getLocalResources().size());
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
    }

    // add resources
    String lrName2 = "LR2";
    lrs.clear();
    lrs.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    client.addAppMasterLocalFiles(lrs);

    ApplicationId appId2 = ApplicationId.newInstance(0, 2);
    when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId())
            .thenReturn(appId2);

    when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState())
            .thenReturn(YarnApplicationState.RUNNING);
    dag = DAG.create("DAG")
            .addVertex(Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)));
    dagClient = client.submitDAG(dag);

    if (isSession) {
        // same app master
        verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
        Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
        // additional resource is sent
        ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
        verify(client.sessionAmProxy, times(2)).submitDAG((RpcController) any(), captor1.capture());
        SubmitDAGRequestProto proto = captor1.getValue();
        Assert.assertEquals(1, proto.getAdditionalAmResources().getLocalResourcesCount());
        Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName());
    } else {
        // new app master
        Assert.assertTrue(dagClient.getExecutionContext().contains(appId2.toString()));
        verify(client.mockYarnClient, times(2)).submitApplication(captor.capture());
        // additional resource is added
        ApplicationSubmissionContext context = captor.getValue();
        Assert.assertEquals(5, context.getAMContainerSpec().getLocalResources().size());
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_PB_BINARY_CONF_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources()
                .containsKey(TezConstants.TEZ_PB_PLAN_BINARY_NAME));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName1));
        Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(lrName2));
    }

    client.stop();
    if (isSession) {
        verify(client.sessionAmProxy, times(1)).shutdownSession((RpcController) any(),
                (ShutdownSessionRequestProto) any());
    }
    verify(client.mockYarnClient, times(1)).stop();
}

From source file:org.apache.tez.client.TestTezClient.java

License:Apache License

public void testMultipleSubmissionsJob(boolean isSession) throws Exception {
    TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession);
    when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState())
            .thenReturn(YarnApplicationState.RUNNING);
    client1.start();//from   ww w. jav a2  s  .co  m

    String mockLR1Name = "LR1";
    Map<String, LocalResource> lrDAG = Collections.singletonMap(mockLR1Name,
            LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE,
                    LocalResourceVisibility.PUBLIC, 1, 1));
    String mockLR2Name = "LR2";
    Map<String, LocalResource> lrVertex = Collections.singletonMap(mockLR2Name,
            LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE,
                    LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1))
            .addTaskLocalFiles(lrVertex);
    DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);

    // the dag resource will be added to the vertex once
    client1.submitDAG(dag);

    TezClientForTest client2 = configure();
    when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState())
            .thenReturn(YarnApplicationState.RUNNING);
    client2.start();

    // verify resubmission of same dag to new client (simulates submission error resulting in the
    // creation of a new client and resubmission of the DAG)
    client2.submitDAG(dag);

    client1.stop();
    client2.stop();
}

From source file:org.apache.tez.client.TestTezClientUtils.java

License:Apache License

public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf)
        throws Exception {

    Path topLevelDir = null;/*from   w  ww.java  2s  . c  om*/
    try {
        FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x
        FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr--
        FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr--
        FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x---

        String fsURI = remoteFs.getUri().toString();

        topLevelDir = new Path(fsURI, "/testLRVisibility");
        Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms));

        Path publicSubDir = new Path(topLevelDir, "public_sub_dir");
        Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms));

        Path privateSubDir = new Path(topLevelDir, "private_sub_dir");
        Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms));

        Path publicFile = new Path(publicSubDir, "public_file");
        Assert.assertTrue(remoteFs.createNewFile(publicFile));
        remoteFs.setPermission(publicFile, publicFilePerms);

        Path privateFile = new Path(publicSubDir, "private_file");
        Assert.assertTrue(remoteFs.createNewFile(privateFile));
        remoteFs.setPermission(privateFile, privateFilePerms);

        Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir");
        Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir));
        remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms);

        TezConfiguration tezConf = new TezConfiguration(conf);
        String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir,
                conf.get(TezConfiguration.TEZ_LIB_URIS, ""));
        tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris);

        Map<String, LocalResource> lrMap = new HashMap<String, LocalResource>();
        TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap);

        Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC,
                lrMap.get(publicFile.getName()).getVisibility());

        Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE,
                lrMap.get(privateFile.getName()).getVisibility());

        Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE,
                lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility());

        // test tar.gz
        tezConf = new TezConfiguration(conf);
        Path tarFile = new Path(topLevelDir, "foo.tar.gz");
        Assert.assertTrue(remoteFs.createNewFile(tarFile));

        //public
        remoteFs.setPermission(tarFile, publicFilePerms);
        tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString());
        lrMap.clear();
        Assert.assertTrue(TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap));

        Assert.assertEquals(LocalResourceVisibility.PUBLIC,
                lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility());

        //private
        remoteFs.setPermission(tarFile, privateFilePerms);
        lrMap.clear();
        TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap);
        Assert.assertEquals(LocalResourceVisibility.PRIVATE,
                lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility());

    } finally {
        if (topLevelDir != null) {
            remoteFs.delete(topLevelDir, true);
        }
    }
}

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

/**
 * Setup LocalResource map for Tez jars based on provided Configuration
 * /*from w  w  w . j  a v a 2 s .  com*/
 * @param conf
 *          Configuration to use to access Tez jars' locations
 * @param credentials
 *          a credentials instance into which tokens for the Tez local
 *          resources will be populated
 * @param tezJarResources Map of LocalResources to use for AM and DAGs
 * @return Whether the archive-based deployment of Tez was used.
 * @throws IOException
 */
static boolean setupTezJarsLocalResources(TezConfiguration conf, Credentials credentials,
        Map<String, LocalResource> tezJarResources) throws IOException {
    Preconditions.checkNotNull(credentials, "A non-null credentials object should be specified");
    boolean usingTezArchive = false;

    if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) {
        LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since  '"
                + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true");
    } else {
        // Add tez jars to local resource
        String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS);

        if (tezJarUris == null || tezJarUris.length == 0) {
            throw new TezUncheckedException("Invalid configuration of tez jars" + ", "
                    + TezConfiguration.TEZ_LIB_URIS + " is not defined in the configuration");
        }

        LOG.info("Using tez.lib.uris value from configuration: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

        if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz")
                || tezJarUris[0].endsWith(".zip") || tezJarUris[0].endsWith(".tar"))) {
            String fileName = tezJarUris[0];

            FileStatus fStatus = getLRFileStatus(fileName, conf)[0];
            LocalResourceVisibility lrVisibility;
            if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE)
                    && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.ARCHIVE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            Path[] tezJarPaths = { fStatus.getPath() };
            // obtain credentials
            TokenCache.obtainTokensForFileSystems(credentials, tezJarPaths, conf);
            usingTezArchive = true;
        } else { // Treat as non-archives
            addLocalResources(conf, tezJarUris, tezJarResources, credentials);
        }

        if (tezJarResources.isEmpty()) {
            throw new TezUncheckedException("No files found in locations specified in "
                    + TezConfiguration.TEZ_LIB_URIS + " . Locations: " + StringUtils.join(tezJarUris, ','));
        }
    }

    // Add aux uris to local resources
    addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS), tezJarResources, credentials);

    return usingTezArchive;
}

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

private static void addLocalResources(Configuration conf, String[] configUris,
        Map<String, LocalResource> tezJarResources, Credentials credentials) throws IOException {
    if (configUris == null || configUris.length == 0) {
        return;/*from  w  w  w  .j av  a 2 s . c o m*/
    }
    List<Path> configuredPaths = Lists.newArrayListWithCapacity(configUris.length);
    for (String configUri : configUris) {
        boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, configUri,
                FsAction.EXECUTE);
        FileStatus[] fileStatuses = getLRFileStatus(configUri, conf);
        for (FileStatus fStatus : fileStatuses) {
            if (fStatus.isDirectory()) {
                // Skip directories - no recursive search support.
                continue;
            }
            LocalResourceVisibility lrVisibility;
            if (ancestorsHavePermission && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            String rsrcName = fStatus.getPath().getName();
            if (tezJarResources.containsKey(rsrcName)) {
                String message = "Duplicate resource found" + ", resourceName=" + rsrcName + ", existingPath="
                        + tezJarResources.get(rsrcName).getResource().toString() + ", newPath="
                        + fStatus.getPath();
                LOG.warn(message);
            }
            tezJarResources.put(rsrcName,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.FILE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            configuredPaths.add(fStatus.getPath());
        }
    }
    // Obtain credentials.
    if (!configuredPaths.isEmpty()) {
        TokenCache.obtainTokensForFileSystems(credentials,
                configuredPaths.toArray(new Path[configuredPaths.size()]), conf);
    }
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testAddAdditionalLocalResources() {
    String lrName = "LR";
    Map<String, LocalResource> originalLrs;
    originalLrs = Maps.newHashMap();//w w w  .  j  a va 2  s. c o m
    originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    Map<String, LocalResource> additionalLrs;

    // Same path, same size.
    originalLrs = Maps.newHashMap();
    originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    additionalLrs = Maps.newHashMap();
    additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, "");

    // Same path, different size.
    originalLrs = Maps.newHashMap();
    originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    additionalLrs = Maps.newHashMap();
    additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1));
    try {
        TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, "");
        Assert.fail("Duplicate LRs with different sizes expected to fail");
    } catch (TezUncheckedException e) {
        Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size"));
    }

    // Different path, same size, diff timestamp
    originalLrs = Maps.newHashMap();
    originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    additionalLrs = Maps.newHashMap();
    additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 100));
    TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, "");

    // Different path, different size
    originalLrs = Maps.newHashMap();
    originalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    additionalLrs = Maps.newHashMap();
    additionalLrs.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1));
    try {
        TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, "");
        Assert.fail("Duplicate LRs with different sizes expected to fail");
    } catch (TezUncheckedException e) {
        Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size"));
    }

}

From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java

License:Apache License

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
    int maxIPCMsgSize = 1024;
    String dagPlanName = "dagplan-name";
    File requestFile = tmpFolder.newFile("request-file");
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);

    byte[] randomBytes = new byte[2 * maxIPCMsgSize];
    (new Random()).nextBytes(randomBytes);
    UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
    Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
    DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);

    String lrName = "localResource";
    String scheme = "file";
    String host = "localhost";
    int port = 80;
    String path = "/test";
    URL lrURL = URL.newInstance(scheme, host, port, path);
    LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE,
            LocalResourceVisibility.PUBLIC, 1, 1);
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(lrName, localResource);

    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan)
            .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
    try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
        requestBuilder.build().writeTo(fileOutputStream);
    }//from  w  w w .  j  ava2 s .co m

    DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
    ACLManager aclManager = mock(ACLManager.class);
    DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(
            new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf)));
    when(dagClientHandler.getACLManager()).thenReturn(aclManager);
    when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id");
    when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);

    requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
    serverImpl.submitDAG(null, requestBuilder.build());

    ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
    verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
    dagPlan = dagPlanCaptor.getValue();
    localResources = localResourcesCaptor.getValue();

    assertEquals(dagPlan.getName(), dagPlanName);
    assertEquals(dagPlan.getVertexCount(), 1);
    assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
    assertArrayEquals(randomBytes,
            dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertEquals(localResources.size(), 1);
    assertTrue(localResources.containsKey(lrName));
    localResource = localResources.get(lrName);
    assertEquals(localResource.getType(), LocalResourceType.FILE);
    assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
    lrURL = localResource.getResource();
    assertEquals(lrURL.getScheme(), scheme);
    assertEquals(lrURL.getHost(), host);
    assertEquals(lrURL.getPort(), port);
    assertEquals(lrURL.getFile(), path);
}

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility) {
    switch (visibility) {
    case PUBLIC:/*from   w w  w .j a  va 2s. c  om*/
        return LocalResourceVisibility.PUBLIC;
    case PRIVATE:
        return LocalResourceVisibility.PRIVATE;
    case APPLICATION:
        return LocalResourceVisibility.APPLICATION;
    default:
        throw new RuntimeException("unknown 'visibility': " + visibility);
    }
}