Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Prototype

LocalResourceVisibility APPLICATION

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Click Source Link

Document

Shared only among containers of the same application on the node.

Usage

From source file:org.apache.metron.maas.service.callback.LaunchContainer.java

License:Apache License

private Map.Entry<String, LocalResource> localizeResource(FileStatus status) {
    URL url = ConverterUtils.getYarnUrlFromURI(status.getPath().toUri());
    LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime());
    String name = status.getPath().getName();
    return new AbstractMap.SimpleEntry<>(name, resource);
}

---

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {// w ww . jav a2  s .co  m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return dst;
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager.java

License:Apache License

public Map<String, LocalResource> getTezResources(Set<String> resourceNames) throws Exception {
    Map<String, LocalResource> tezResources = new HashMap<String, LocalResource>();
    for (String resourceName : resourceNames) {
        // The resource name will be symlinked to the resource path in the
        // container's working directory.
        Path resourcePath = resources.get(resourceName);
        FileStatus fstat = remoteFs.getFileStatus(resourcePath);

        LocalResource tezResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromPath(fstat.getPath()), LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION, fstat.getLen(), fstat.getModificationTime());

        tezResources.put(resourceName, tezResource);
    }//from ww w .  j a  va2 s .  co  m
    return tezResources;
}

---

From source file:org.apache.reef.runtime.yarn.client.uploader.JobFolder.java

License:Apache License

/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path.
 *///from   www .  jav  a 2s  . c  o m
public LocalResource getLocalResourceForPath(final Path jarPath) throws IOException {
    final LocalResource localResource = Records.newRecord(LocalResource.class);
    final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath);
    localResource.setType(LocalResourceType.ARCHIVE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
    localResource.setTimestamp(status.getModificationTime());
    localResource.setSize(status.getLen());
    return localResource;
}

---

From source file:org.apache.reef.runtime.yarn.client.YarnJobSubmissionHandler.java

License:Apache License

/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path
 *//*from ww w .jav  a  2 s .  co m*/
private LocalResource makeLocalResourceForJarFile(final Path path) throws IOException {
    final LocalResource localResource = Records.newRecord(LocalResource.class);
    final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(path);
    localResource.setType(LocalResourceType.ARCHIVE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
    localResource.setTimestamp(status.getModificationTime());
    localResource.setSize(status.getLen());
    return localResource;
}

---

From source file:org.apache.reef.runtime.yarn.driver.UploaderToJobFolder.java

License:Apache License

/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path
 *
 * @param path//from   ww  w  . j a  va 2 s . c  o m
 * @return
 * @throws IOException
 */
LocalResource makeLocalResourceForJarFile(final Path path) throws IOException {
    final LocalResource localResource = Records.newRecord(LocalResource.class);
    final FileStatus status = FileContext.getFileContext(this.fileSystem.getUri()).getFileStatus(path);
    localResource.setType(LocalResourceType.ARCHIVE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
    localResource.setTimestamp(status.getModificationTime());
    localResource.setSize(status.getLen());
    return localResource;
}

---

From source file:org.apache.samza.job.yarn.ContainerUtil.java

License:Apache License

protected void startContainer(Path packagePath, Container container, Map<String, String> env,
        final String cmd) {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;/*w w w  .  j a va  2s .  co  m*/
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaException("IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        ioe.printStackTrace();
        throw new SamzaException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaException("Received YarnException when starting container: " + container.getId());
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaException("Received IOException when starting container: " + container.getId());
    }
}

---

From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java

License:Apache License

/**
 *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 *    specified by packagePath.//from   ww w.j a v a  2 s  .  c  o  m
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws SamzaContainerLaunchException {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaContainerLaunchException(
                "IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        log.error("IOException when writing credentials.", ioe);
        throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaContainerLaunchException(
                "Received YarnException when starting container: " + container.getId(), ye);
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaContainerLaunchException(
                "Received IOException when starting container: " + container.getId(), ioe);
    }
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapSuccess() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");

    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");

    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);//from  w  w  w .ja  va 2 s.  c  om
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());

    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());

    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());

    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());

    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either",
            resourceMap.get("notExisting"));
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithDefaultValues() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);//w ww  .  j  a  v a 2 s . c om
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
    assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
    assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »