Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PRIVATE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PRIVATE.

Prototype

LocalResourceVisibility PRIVATE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility PRIVATE.

Click Source Link

Document

Shared among all applications of the same user on the node.

Usage

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

private static void addLocalResources(Configuration conf, String[] configUris,
        Map<String, LocalResource> tezJarResources, Credentials credentials) throws IOException {
    if (configUris == null || configUris.length == 0) {
        return;/*from w  w w . j  ava2s  .com*/
    }
    List<Path> configuredPaths = Lists.newArrayListWithCapacity(configUris.length);
    for (String configUri : configUris) {
        boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, configUri,
                FsAction.EXECUTE);
        FileStatus[] fileStatuses = getLRFileStatus(configUri, conf);
        for (FileStatus fStatus : fileStatuses) {
            if (fStatus.isDirectory()) {
                // Skip directories - no recursive search support.
                continue;
            }
            LocalResourceVisibility lrVisibility;
            if (ancestorsHavePermission && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
                lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
                lrVisibility = LocalResourceVisibility.PRIVATE;
            }
            String rsrcName = fStatus.getPath().getName();
            if (tezJarResources.containsKey(rsrcName)) {
                String message = "Duplicate resource found" + ", resourceName=" + rsrcName + ", existingPath="
                        + tezJarResources.get(rsrcName).getResource().toString() + ", newPath="
                        + fStatus.getPath();
                LOG.warn(message);
            }
            tezJarResources.put(rsrcName,
                    LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                            LocalResourceType.FILE, lrVisibility, fStatus.getLen(),
                            fStatus.getModificationTime()));
            configuredPaths.add(fStatus.getPath());
        }
    }
    // Obtain credentials.
    if (!configuredPaths.isEmpty()) {
        TokenCache.obtainTokensForFileSystems(credentials,
                configuredPaths.toArray(new Path[configuredPaths.size()]), conf);
    }
}

---

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility) {
    switch (visibility) {
    case PUBLIC://from  ww  w. ja v  a 2  s.  co m
        return LocalResourceVisibility.PUBLIC;
    case PRIVATE:
        return LocalResourceVisibility.PRIVATE;
    case APPLICATION:
        return LocalResourceVisibility.APPLICATION;
    default:
        throw new RuntimeException("unknown 'visibility': " + visibility);
    }
}

---

From source file:org.apache.tez.dag.app.launcher.TestTezLocalCacheManager.java

License:Apache License

private static LocalResource createFile(String content) throws IOException {
    FileContext fs = FileContext.getLocalFSFileContext();

    java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
    File temp = tempFile.toFile();
    temp.deleteOnExit();/*ww  w.  j a  va2 s . c  om*/
    Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

    Files.write(tempFile, content.getBytes());

    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
    ret.setResource(yarnUrlFromPath);
    ret.setSize(content.getBytes().length);
    ret.setType(LocalResourceType.FILE);
    ret.setVisibility(LocalResourceVisibility.PRIVATE);
    ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
    return ret;
}

---

From source file:org.apache.tez.mapreduce.examples.RPCLoadGen.java

License:Apache License

private UserPayload createUserPayload(TezConfiguration conf, int maxSleepTimeMillis, int payloadSize,
        String mode, Map<String, LocalResource> localResources) throws IOException {
    ByteBuffer payload;// ww w  .  j av a2 s . c  om
    if (mode.equals(VIA_RPC)) {
        if (payloadSize < 5) {
            payloadSize = 5; // To Configure the processor
        }
        byte[] payloadBytes = new byte[payloadSize];
        random.nextBytes(payloadBytes);
        payload = ByteBuffer.wrap(payloadBytes);
        payload.put(4, VIA_RPC_BYTE); // ViaRPC
    } else {
        // Actual payload
        byte[] payloadBytes = new byte[5];
        payload = ByteBuffer.wrap(payloadBytes);

        // Disk payload
        byte[] diskPayload = new byte[payloadSize];
        random.nextBytes(diskPayload);
        fs = FileSystem.get(conf);
        resourcePath = new Path(Path.SEPARATOR + "tmp", DISK_PAYLOAD_NAME);
        System.err.println("ZZZ: HDFSPath: " + resourcePath);
        resourcePath = fs.makeQualified(resourcePath);
        System.err.println("ZZZ: HDFSPathResolved: " + resourcePath);
        FSDataOutputStream dataOut = fs.create(resourcePath, true);
        dataOut.write(diskPayload);
        dataOut.close();
        fs.setReplication(resourcePath, (short) 10);
        FileStatus fileStatus = fs.getFileStatus(resourcePath);

        if (mode.equals(VIA_HDFS_DIST_CACHE)) {
            LocalResource lr = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(resourcePath),
                    LocalResourceType.ARCHIVE.FILE, LocalResourceVisibility.PRIVATE, fileStatus.getLen(),
                    fileStatus.getModificationTime());
            localResources.put(DISK_PAYLOAD_NAME, lr);
            payload.put(4, VIA_HDFS_DIST_CACHE_BYTE); // ViaRPC
        } else if (mode.equals(VIA_HDFS_DIRECT_READ)) {
            payload.put(4, VIA_HDFS_DIRECT_READ_BYTE); // ViaRPC
        }
    }

    payload.putInt(0, maxSleepTimeMillis);
    return UserPayload.create(payload);
}

---

From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java

License:Apache License

private LocalResource createLrObjFromPath(Path filePath) {
    return LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(filePath), LocalResourceType.FILE,
            LocalResourceVisibility.PRIVATE, 0, 0);
}

---

From source file:org.springframework.yarn.config.LocalresourcesNamespaceTest.java

License:Apache License

@Test
public void testGlobalDefaults() throws Exception {
    assertNotNull(resourcesWithGlobal);//from  w w w .  j av a  2  s.c o m
    assertNotNull(localResourcesFactoryBeanWithGlobal);
    Configuration configuration = TestUtils.readField("configuration", localResourcesFactoryBeanWithGlobal);
    assertNotNull(configuration);

    Collection<TransferEntry> hdfsEntries = TestUtils.readField("hdfsEntries",
            localResourcesFactoryBeanWithGlobal);
    assertNotNull(hdfsEntries);
    assertThat(hdfsEntries.size(), is(1));

    TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0];
    String path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/foo.jar"));
    LocalResourceType type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.ARCHIVE));
    LocalResourceVisibility visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PRIVATE));
    String local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9000"));
    String remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9000"));
}

---

From source file:org.springframework.yarn.config.LocalresourcesNamespaceTest.java

License:Apache License

@Test
public void testGlobalLocalMixed() throws Exception {
    assertNotNull(resourcesWithMixed);/*from   w  w w .j av  a  2s .c o  m*/
    assertNotNull(localResourcesFactoryBeanWithMixed);
    Configuration configuration = TestUtils.readField("configuration", localResourcesFactoryBeanWithMixed);
    assertNotNull(configuration);

    Collection<TransferEntry> hdfsEntries = TestUtils.readField("hdfsEntries",
            localResourcesFactoryBeanWithMixed);
    assertNotNull(hdfsEntries);
    assertThat(hdfsEntries.size(), is(3));

    TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0];
    String path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/foo.jar"));
    LocalResourceType type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    LocalResourceVisibility visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PUBLIC));
    String local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9001"));
    String remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9000"));

    entry = (TransferEntry) hdfsEntries.toArray()[1];
    path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/jee.jar"));
    type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PUBLIC));
    local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9000"));
    remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9001"));

    entry = (TransferEntry) hdfsEntries.toArray()[2];
    path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/bar.jar"));
    type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.ARCHIVE));
    visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PRIVATE));
    local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9000"));
    remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9000"));
}

---

• «
• 1
• 2
• »