Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC.

Prototype

LocalResourceVisibility PUBLIC

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC.

Click Source Link

Document

Shared by all users on the node.

Usage

From source file:org.apache.tez.dag.api.TestDAGVerify.java

License:Apache License

@Test(timeout = 5000)
public void testDAGCreateDataInference() {
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
    Map<String, LocalResource> lrs1 = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Map<String, LocalResource> lrs2 = Maps.newHashMap();
    String lrName2 = "LR2";
    lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    Set<String> hosts = Sets.newHashSet();
    hosts.add("h1");
    hosts.add("h2");
    List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
    DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"),
            InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
    v1.addDataSource("i1", ds);

    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);/*from w w w  .j  a v  a  2s  .c  om*/
    dag.addTaskLocalFiles(lrs1);
    DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
    Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
    VertexPlan vPlan = dagPlan.getVertex(0);
    PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
    Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
    Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
    Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
    Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}

---

From source file:org.apache.tez.dag.api.TestDAGVerify.java

License:Apache License

@Test(timeout = 5000)
public void testInferredFilesFail() {
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName), dummyTaskCount,
            dummyTaskResource);/* ww w  .  ja v a2 s  .  com*/
    Map<String, LocalResource> lrs = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    // Same lr, different size
    Map<String, LocalResource> lrs2 = Maps.newHashMap();
    lrs2.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 100, 1));

    v1.addTaskLocalFiles(lrs);
    // Allowed since the LR is the same.
    try {
        v1.addTaskLocalFiles(lrs2);
        Assert.fail();
    } catch (TezUncheckedException e) {
        Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size"));
    }

    DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), null, -1, null,
            null, lrs2);
    v1.addDataSource("i1", ds);

    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);
    dag.addTaskLocalFiles(lrs);
    try {
        dag.addTaskLocalFiles(lrs2);
        Assert.fail();
    } catch (TezUncheckedException e) {
        Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size"));
    }
    try {
        // data source will add duplicate common files to vertex
        dag.createDag(new TezConfiguration(), null, null, null, true);
        Assert.fail();
    } catch (TezUncheckedException e) {
        Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size"));
    }
}

---

From source file:org.apache.tez.dag.app.TestMockDAGAppMaster.java

License:Apache License

@Test(timeout = 5000)
public void testLocalResourceSetup() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);

    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();//from w w  w. ja v a 2s .c o  m

    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);

    Map<String, LocalResource> lrDAG = Maps.newHashMap();
    String lrName1 = "LR1";
    lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Map<String, LocalResource> lrVertex = Maps.newHashMap();
    String lrName2 = "LR2";
    lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
    dag.addVertex(vA);

    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    ContainerData cData = mockLauncher.getContainers().values().iterator().next();
    ContainerLaunchContext launchContext = cData.launchContext;
    Map<String, LocalResource> taskLR = launchContext.getLocalResources();
    // verify tasks are launched with both DAG and task resources.
    Assert.assertTrue(taskLR.containsKey(lrName1));
    Assert.assertTrue(taskLR.containsKey(lrName2));

    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    tezClient.stop();
}

---

From source file:org.apache.tez.dag.app.TestMockDAGAppMaster.java

License:Apache License

@Test(timeout = 10000)
public void testMultipleSubmissions() throws Exception {
    Map<String, LocalResource> lrDAG = Maps.newHashMap();
    String lrName1 = "LR1";
    lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Map<String, LocalResource> lrVertex = Maps.newHashMap();
    String lrName2 = "LR2";
    lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
            LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

    DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
    dag.addVertex(vA);//w ww .j  a v a2  s .  co m

    TezConfiguration tezconf = new TezConfiguration(defaultConf);

    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    DAGClient dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    tezClient.stop();

    // submit the same DAG again to verify it can be done.
    tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    tezClient.stop();
}

---

From source file:org.elasticsearch.hadoop.yarn.am.EsCluster.java

License:Apache License

private Map<String, LocalResource> setupEsZipResource(Config conf) {
    // elasticsearch.zip
    Map<String, LocalResource> resources = new LinkedHashMap<String, LocalResource>();

    LocalResource esZip = Records.newRecord(LocalResource.class);
    String esZipHdfsPath = conf.esZipHdfsPath();
    Path p = new Path(esZipHdfsPath);
    FileStatus fsStat;/*ww  w .j  a va  2  s . c om*/
    try {
        fsStat = FileSystem.get(cfg).getFileStatus(p);
    } catch (IOException ex) {
        throw new IllegalArgumentException(String.format(
                "Cannot find Elasticsearch zip at [%s]; make sure the artifacts have been properly provisioned and the correct permissions are in place.",
                esZipHdfsPath), ex);
    }
    // use the normalized path as otherwise YARN chokes down the line
    esZip.setResource(ConverterUtils.getYarnUrlFromPath(fsStat.getPath()));
    esZip.setSize(fsStat.getLen());
    esZip.setTimestamp(fsStat.getModificationTime());
    esZip.setType(LocalResourceType.ARCHIVE);
    esZip.setVisibility(LocalResourceVisibility.PUBLIC);

    resources.put(conf.esZipName(), esZip);
    return resources;
}

---

From source file:org.elasticsearch.hadoop.yarn.client.YarnLauncher.java

License:Apache License

private Map<String, LocalResource> setupEsYarnJar() {
    Map<String, LocalResource> resources = new LinkedHashMap<String, LocalResource>();
    LocalResource esYarnJar = Records.newRecord(LocalResource.class);
    Path p = new Path(clientCfg.jarHdfsPath());
    FileStatus fsStat;//from   w  w w .ja v  a  2s . com
    try {
        fsStat = FileSystem.get(client.getConfiguration()).getFileStatus(p);
    } catch (IOException ex) {
        throw new IllegalArgumentException(String.format(
                "Cannot find jar [%s]; make sure the artifacts have been properly provisioned and the correct permissions are in place.",
                clientCfg.jarHdfsPath()), ex);
    }
    // use the normalized path as otherwise YARN chokes down the line
    esYarnJar.setResource(ConverterUtils.getYarnUrlFromPath(fsStat.getPath()));
    esYarnJar.setSize(fsStat.getLen());
    esYarnJar.setTimestamp(fsStat.getModificationTime());
    esYarnJar.setType(LocalResourceType.FILE);
    esYarnJar.setVisibility(LocalResourceVisibility.PUBLIC);

    resources.put(clientCfg.jarName(), esYarnJar);
    return resources;
}

---

From source file:org.springframework.yarn.config.LocalresourcesNamespaceTest.java

License:Apache License

@Test
public void testGlobalLocalMixed() throws Exception {
    assertNotNull(resourcesWithMixed);/*from w  w  w  .  j  av  a2 s . com*/
    assertNotNull(localResourcesFactoryBeanWithMixed);
    Configuration configuration = TestUtils.readField("configuration", localResourcesFactoryBeanWithMixed);
    assertNotNull(configuration);

    Collection<TransferEntry> hdfsEntries = TestUtils.readField("hdfsEntries",
            localResourcesFactoryBeanWithMixed);
    assertNotNull(hdfsEntries);
    assertThat(hdfsEntries.size(), is(3));

    TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0];
    String path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/foo.jar"));
    LocalResourceType type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    LocalResourceVisibility visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PUBLIC));
    String local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9001"));
    String remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9000"));

    entry = (TransferEntry) hdfsEntries.toArray()[1];
    path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/jee.jar"));
    type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PUBLIC));
    local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9000"));
    remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9001"));

    entry = (TransferEntry) hdfsEntries.toArray()[2];
    path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/bar.jar"));
    type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.ARCHIVE));
    visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PRIVATE));
    local = TestUtils.readField("local", entry);
    assertThat(local, is("hdfs://0.0.0.0:9000"));
    remote = TestUtils.readField("remote", entry);
    assertThat(remote, is("hdfs://192.168.223.139:9000"));
}

---

From source file:tachyon.yarn.Client.java

License:Apache License

private void setupAppMasterJar(LocalResource appMasterJar) throws IOException {
    Path jarHdfsPath = new Path(mAppMasterJarHdfs); // known path to jar file on HDFS
    FileStatus jarStat = FileSystem.get(mYarnConf).getFileStatus(jarHdfsPath);
    appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarHdfsPath));
    appMasterJar.setSize(jarStat.getLen());
    appMasterJar.setTimestamp(jarStat.getModificationTime());
    appMasterJar.setType(LocalResourceType.FILE);
    appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC);
}

---

From source file:tachyon.yarn.Utils.java

License:Apache License

/**
 * Creates a local resource for a file on HDFS.
 *
 * @param yarnConf YARN configuration//  w  w w .  j a v a2s. co  m
 * @param resourcePath known path of resource file on HDFS
 * @throws IOException if the file can not be found on HDFS
 */
public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resourcePath)
        throws IOException {
    LocalResource localResource = Records.newRecord(LocalResource.class);

    Path jarHdfsPath = new Path(resourcePath);
    FileStatus jarStat = FileSystem.get(yarnConf).getFileStatus(jarHdfsPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarHdfsPath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
    return localResource;
}

---

From source file:yrun.YarnRunner.java

License:Apache License

private void setupAppMasterArgs(Path yarnRunnerArgsPath, LocalResource appMasterArgs) throws IOException {
    FileSystem fileSystem = yarnRunnerArgsPath.getFileSystem(_configuration);
    FileStatus jarStatus = fileSystem.getFileStatus(yarnRunnerArgsPath);
    appMasterArgs.setResource(ConverterUtils.getYarnUrlFromPath(yarnRunnerArgsPath));
    appMasterArgs.setSize(jarStatus.getLen());
    appMasterArgs.setTimestamp(jarStatus.getModificationTime());
    appMasterArgs.setType(LocalResourceType.FILE);
    appMasterArgs.setVisibility(LocalResourceVisibility.PUBLIC);
}

---

• «
• 1
• 2
• 3
• 4
• »