Example usage for org.apache.hadoop.yarn.api.records LocalResourceType ARCHIVE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceType ARCHIVE.

Prototype

LocalResourceType ARCHIVE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType ARCHIVE.

Click Source Link

Document

Archive, which is automatically unarchived by the NodeManager.

Usage

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupLocalResources(ContainerLaunchContext amContainer, FileSystem fs, ApplicationId appId)
        throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path

    String appMasterJar = findContainingJar(ApplicationMaster.class);
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources,
            LocalResourceType.FILE);/*from w  ww.ja  va  2 s. c o m*/

    addToLocalResources(fs, libDir, libDir, appId.getId(), localResources, LocalResourceType.FILE);

    addToLocalResources(fs, tajoArchive, "tajo", appId.getId(), localResources, LocalResourceType.ARCHIVE);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources,
                LocalResourceType.FILE);
    }

    //    addToLocalResources(fs, confDir, "conf", appId.getId(),
    //        localResources, LocalResourceType.FILE);

    // Tajo master conf
    Configuration tajoMasterConf = new Configuration(false);
    tajoMasterConf.addResource(new Path(confDir, "tajo-site.xml"));
    String suffix = appName + "/" + appId.getId() + "/master-conf";
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoMasterConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);

    amContainer.setLocalResources(localResources);
}

---

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

@Override
public ContainerLaunchContext getLaunchContext(Container container) throws IOException {
    // create a container launch context
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
    UserGroupInformation user = UserGroupInformation.getCurrentUser();
    try {/*  w  w w .j av a2  s .c o  m*/
        Credentials credentials = user.getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        launchContext.setTokens(securityTokens);
    } catch (IOException e) {
        LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    FileSystem fs = FileSystem.get(appContext.getConfiguration());

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    String suffix = "Tajo" + "/" + appContext.getApplicationId().getId();
    Path parentPath = new Path(fs.getHomeDirectory(), suffix);

    // tar ball
    Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH));
    FileStatus archiveFs = fs.getFileStatus(archivePath);
    LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(),
            archiveFs.getModificationTime());
    localResources.put("tajo", archiveRsrc);

    Configuration tajoWorkerConf = new Configuration(false);
    tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml"));
    tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001");
    tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005");
    Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf");
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoWorkerConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);
    launchContext.setLocalResources(localResources);

    // Set the environment
    setupEnv(launchContext);

    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);

    // Set executable command
    // Set args for the shell command if any
    vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo");
    vargs.add("--config");
    vargs.add("${" + Constants.TAJO_CONF_DIR + "}");
    vargs.add("worker");
    // Add log redirect params
    // Add log redirect params
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    launchContext.setCommands(commands);
    return launchContext;
}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

/**
 * Setup LocalResource map for Tez jars based on provided Configuration
 * //from w w w.  j av a2 s. c o  m
 * @param conf
 *          Configuration to use to access Tez jars' locations
 * @param credentials
 *          a credentials instance into which tokens for the Tez local
 *          resources will be populated
 * @param tezJarResources Map of LocalResources to use for AM and DAGs
 * @return Whether the archive-based deployment of Tez was used.
 * @throws IOException
 */
static boolean setupTezJarsLocalResources(TezConfiguration conf, Credentials credentials,
        Map<String, LocalResource> tezJarResources) throws IOException {
    Preconditions.checkNotNull(credentials, "A non-null credentials object should be specified");
    boolean usingTezArchive = false;

    if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) {
        LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since  '"
                + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true");
    } else {
        // Add tez jars to local resource
        String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS);

        if (tezJarUris == null || tezJarUris.length == 0) {
            throw new TezUncheckedException("Invalid configuration of tez jars" + ", "
                    + TezConfiguration.TEZ_LIB_URIS + " is not defined in the configuration");
        }

        LOG.info("Using tez.lib.uris value from configuration: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

        if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz")
                || tezJarUris[0].endsWith(".zip") || tezJarUris[0].endsWith(".tar"))) {
            String fileName = tezJarUris[0];

            FileStatus fStatus = getLRFileStatus(fileName, conf)[0];
            LocalResourceVisibility lrVisibility;
            if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE)
                    && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.ARCHIVE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            Path[] tezJarPaths = { fStatus.getPath() };
            // obtain credentials
            TokenCache.obtainTokensForFileSystems(credentials, tezJarPaths, conf);
            usingTezArchive = true;
        } else { // Treat as non-archives
            addLocalResources(conf, tezJarUris, tezJarResources, credentials);
        }

        if (tezJarResources.isEmpty()) {
            throw new TezUncheckedException("No files found in locations specified in "
                    + TezConfiguration.TEZ_LIB_URIS + " . Locations: " + StringUtils.join(tezJarUris, ','));
        }
    }

    // Add aux uris to local resources
    addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS), tezJarResources, credentials);

    return usingTezArchive;
}

---

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
    switch (type) {
    case ARCHIVE:
        return LocalResourceType.ARCHIVE;
    case FILE://from ww  w.  j  a v a2s .c  o m
        return LocalResourceType.FILE;
    case PATTERN:
        return LocalResourceType.PATTERN;
    default:
        throw new IllegalArgumentException("unknown 'type': " + type);
    }
}

---

From source file:org.apache.tez.dag.app.ContainerContext.java

License:Apache License

private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
        Map<String, LocalResource> reqLRs) {
    Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
    for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
        LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
        if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Cannot match container: Attempting to use same target resource name: "
                        + srcLREntry.getKey() + ", but with different source resources. Already localized: "
                        + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
            }/*  w w w.jav  a 2 s .c  o m*/
            return false;
        }
    }
    for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
        LocalResource lr = additionalLREntry.getValue();
        if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
            return false;
        }
    }
    return true;
}

---

From source file:org.apache.tez.mapreduce.examples.RPCLoadGen.java

License:Apache License

private UserPayload createUserPayload(TezConfiguration conf, int maxSleepTimeMillis, int payloadSize,
        String mode, Map<String, LocalResource> localResources) throws IOException {
    ByteBuffer payload;//from  ww  w .  j  a v a 2  s .com
    if (mode.equals(VIA_RPC)) {
        if (payloadSize < 5) {
            payloadSize = 5; // To Configure the processor
        }
        byte[] payloadBytes = new byte[payloadSize];
        random.nextBytes(payloadBytes);
        payload = ByteBuffer.wrap(payloadBytes);
        payload.put(4, VIA_RPC_BYTE); // ViaRPC
    } else {
        // Actual payload
        byte[] payloadBytes = new byte[5];
        payload = ByteBuffer.wrap(payloadBytes);

        // Disk payload
        byte[] diskPayload = new byte[payloadSize];
        random.nextBytes(diskPayload);
        fs = FileSystem.get(conf);
        resourcePath = new Path(Path.SEPARATOR + "tmp", DISK_PAYLOAD_NAME);
        System.err.println("ZZZ: HDFSPath: " + resourcePath);
        resourcePath = fs.makeQualified(resourcePath);
        System.err.println("ZZZ: HDFSPathResolved: " + resourcePath);
        FSDataOutputStream dataOut = fs.create(resourcePath, true);
        dataOut.write(diskPayload);
        dataOut.close();
        fs.setReplication(resourcePath, (short) 10);
        FileStatus fileStatus = fs.getFileStatus(resourcePath);

        if (mode.equals(VIA_HDFS_DIST_CACHE)) {
            LocalResource lr = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(resourcePath),
                    LocalResourceType.ARCHIVE.FILE, LocalResourceVisibility.PRIVATE, fileStatus.getLen(),
                    fileStatus.getModificationTime());
            localResources.put(DISK_PAYLOAD_NAME, lr);
            payload.put(4, VIA_HDFS_DIST_CACHE_BYTE); // ViaRPC
        } else if (mode.equals(VIA_HDFS_DIRECT_READ)) {
            payload.put(4, VIA_HDFS_DIRECT_READ_BYTE); // ViaRPC
        }
    }

    payload.putInt(0, maxSleepTimeMillis);
    return UserPayload.create(payload);
}

---

From source file:org.apache.twill.internal.yarn.YarnUtils.java

License:Apache License

private static YarnLocalResource setLocalResourceType(YarnLocalResource localResource, LocalFile localFile) {
    if (localFile.isArchive()) {
        if (localFile.getPattern() == null) {
            localResource.setType(LocalResourceType.ARCHIVE);
        } else {/*  w  w  w.ja va 2 s.co  m*/
            localResource.setType(LocalResourceType.PATTERN);
            localResource.setPattern(localFile.getPattern());
        }
    } else {
        localResource.setType(LocalResourceType.FILE);
    }
    return localResource;
}

---

From source file:org.elasticsearch.hadoop.yarn.am.EsCluster.java

License:Apache License

private Map<String, LocalResource> setupEsZipResource(Config conf) {
    // elasticsearch.zip
    Map<String, LocalResource> resources = new LinkedHashMap<String, LocalResource>();

    LocalResource esZip = Records.newRecord(LocalResource.class);
    String esZipHdfsPath = conf.esZipHdfsPath();
    Path p = new Path(esZipHdfsPath);
    FileStatus fsStat;/*from w w  w.  j  av a 2  s  . c  o m*/
    try {
        fsStat = FileSystem.get(cfg).getFileStatus(p);
    } catch (IOException ex) {
        throw new IllegalArgumentException(String.format(
                "Cannot find Elasticsearch zip at [%s]; make sure the artifacts have been properly provisioned and the correct permissions are in place.",
                esZipHdfsPath), ex);
    }
    // use the normalized path as otherwise YARN chokes down the line
    esZip.setResource(ConverterUtils.getYarnUrlFromPath(fsStat.getPath()));
    esZip.setSize(fsStat.getLen());
    esZip.setTimestamp(fsStat.getModificationTime());
    esZip.setType(LocalResourceType.ARCHIVE);
    esZip.setVisibility(LocalResourceVisibility.PUBLIC);

    resources.put(conf.esZipName(), esZip);
    return resources;
}

---

From source file:org.springframework.yarn.boot.support.BootLocalResourcesSelectorTests.java

License:Apache License

@Test
public void testAppmasterModeDefault() {
    BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.APPMASTER);
    List<Entry> entries = selector.select("");
    assertThat(entries, notNullValue());
    assertThat(entries.size(), is(4));//  w  ww . j a v a  2  s .c  o  m

    Entry entry = findEntry("application.yml", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("application.properties", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("*appmaster*jar", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("*appmaster*zip", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), is(LocalResourceType.ARCHIVE));
}

---

From source file:org.springframework.yarn.boot.support.BootLocalResourcesSelectorTests.java

License:Apache License

@Test
public void testContainerModeDefault() {
    BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.CONTAINER);
    List<Entry> entries = selector.select("");
    assertThat(entries, notNullValue());
    assertThat(entries.size(), is(4));/*from   ww w .j a  v  a 2s  .c o m*/

    Entry entry = findEntry("application.yml", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("application.properties", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("*container*jar", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), nullValue());

    entry = findEntry("*container*zip", entries);
    assertThat(entry, notNullValue());
    assertThat(entry.getType(), is(LocalResourceType.ARCHIVE));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »