Example usage for org.apache.hadoop.yarn.api.records LocalResource getVisibility

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource getVisibility.

Prototype

@Public
@Stable
public abstract LocalResourceVisibility getVisibility();

Source Link

Document

Get the LocalResourceVisibility of the resource to be localized.

Usage

From source file:org.apache.drill.yarn.core.LaunchSpec.java

License:Apache License

public void dump(PrintStream out) {
    if (command != null) {
        out.print("Command: ");
        out.println(command);//  w  w w  .j  a v  a  2 s.co m
    }
    if (mainClass != null) {
        out.print("Main Class: ");
        out.println(mainClass);
        out.println("VM Args:");
        if (vmArgs.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : vmArgs) {
                out.print("  ");
                out.println(item);
            }
        }
        out.println("Class Path:");
        if (classPath.isEmpty()) {
            out.println("  None");
        } else {
            for (String item : classPath) {
                out.print("  ");
                out.println(item);
            }
        }
    }
    out.println("Program Args:");
    if (cmdArgs.isEmpty()) {
        out.println("  None");
    } else {
        for (String item : cmdArgs) {
            out.print("  ");
            out.println(item);
        }
    }
    out.println("Environment:");
    if (env.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : env.keySet()) {
            out.print("  ");
            out.print(key);
            out.print("=");
            out.println(env.get(key));
        }
    }
    out.println("Resources: ");
    if (resources.isEmpty()) {
        out.println("  None");
    } else {
        for (String key : resources.keySet()) {
            out.print("  Key: ");
            out.println(key);
            LocalResource resource = resources.get(key);
            out.print("   URL: ");
            out.println(resource.getResource().toString());
            out.print("   Size: ");
            out.println(resource.getSize());
            out.print("   Timestamp: ");
            out.println(DoYUtil.toIsoTime(resource.getTimestamp()));
            out.print("   Type: ");
            out.println(resource.getType().toString());
            out.print("   Visiblity: ");
            out.println(resource.getVisibility().toString());
        }
    }
}

---

From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java

License:Apache License

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
    int maxIPCMsgSize = 1024;
    String dagPlanName = "dagplan-name";
    File requestFile = tmpFolder.newFile("request-file");
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);

    byte[] randomBytes = new byte[2 * maxIPCMsgSize];
    (new Random()).nextBytes(randomBytes);
    UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
    Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
    DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);

    String lrName = "localResource";
    String scheme = "file";
    String host = "localhost";
    int port = 80;
    String path = "/test";
    URL lrURL = URL.newInstance(scheme, host, port, path);
    LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE,
            LocalResourceVisibility.PUBLIC, 1, 1);
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(lrName, localResource);

    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan)
            .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
    try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
        requestBuilder.build().writeTo(fileOutputStream);
    }/*from   w w w  .  j  av  a  2 s. c o  m*/

    DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
    ACLManager aclManager = mock(ACLManager.class);
    DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(
            new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf)));
    when(dagClientHandler.getACLManager()).thenReturn(aclManager);
    when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id");
    when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);

    requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
    serverImpl.submitDAG(null, requestBuilder.build());

    ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
    verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
    dagPlan = dagPlanCaptor.getValue();
    localResources = localResourcesCaptor.getValue();

    assertEquals(dagPlan.getName(), dagPlanName);
    assertEquals(dagPlan.getVertexCount(), 1);
    assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
    assertArrayEquals(randomBytes,
            dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertEquals(localResources.size(), 1);
    assertTrue(localResources.containsKey(lrName));
    localResource = localResources.get(lrName);
    assertEquals(localResource.getType(), LocalResourceType.FILE);
    assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
    lrURL = localResource.getResource();
    assertEquals(lrURL.getScheme(), scheme);
    assertEquals(lrURL.getHost(), host);
    assertEquals(lrURL.getPort(), port);
    assertEquals(lrURL.getFile(), path);
}

---

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) {
    List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size());
    for (Entry<String, LocalResource> entry : lrs.entrySet()) {
        PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
        String key = entry.getKey();
        LocalResource lr = entry.getValue();
        localResourcesBuilder.setName(key);
        localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
        localResourcesBuilder.setSize(lr.getSize());
        localResourcesBuilder.setTimeStamp(lr.getTimestamp());
        localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
        localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
        if (lr.getType() == LocalResourceType.PATTERN) {
            if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
                throw new TezUncheckedException(
                        "LocalResource type set to pattern" + " but pattern is null or empty");
            }/*from w  ww.j  av  a 2 s. c o  m*/
            localResourcesBuilder.setPattern(lr.getPattern());
        }
        planLrs.add(localResourcesBuilder.build());
    }
    return planLrs;
}

---

From source file:org.apache.tez.dag.api.DagTypeConverters.java

License:Apache License

public static PlanLocalResource convertLocalResourceToPlanLocalResource(String name, LocalResource lr) {
    PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
    localResourcesBuilder.setName(name);
    localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
    localResourcesBuilder.setSize(lr.getSize());
    localResourcesBuilder.setTimeStamp(lr.getTimestamp());
    localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
    localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
    if (lr.getType() == LocalResourceType.PATTERN) {
        if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
            throw new TezUncheckedException(
                    "LocalResource type set to pattern" + " but pattern is null or empty");
        }//from   w  ww  . j  a v a 2s.c  o m
        localResourcesBuilder.setPattern(lr.getPattern());
    }
    return localResourcesBuilder.build();
}

---