Example usage for org.apache.hadoop.yarn.util Records newRecord

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.util Records newRecord.

Prototype

public static <T> T newRecord(Class<T> cls) 

Source Link

Usage

From source file:UnmanagedAMLauncher.java

License:Apache License

public boolean run() throws IOException, YarnException {
    LOG.info("Starting Client");

    // Connect to ResourceManager
    rmClient.start();/*from  w  ww.  j a v a  2 s .co  m*/
    try {
        // Create launch context for app master
        LOG.info("Setting up application submission context for ASM");
        ApplicationSubmissionContext appContext = rmClient.createApplication()
                .getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();

        // set the application name
        appContext.setApplicationName(appName);

        // Set the priority for the application master
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(amPriority);
        appContext.setPriority(pri);

        // Set the queue to which this application is to be submitted in the RM
        appContext.setQueue(amQueue);

        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        appContext.setAMContainerSpec(amContainer);

        // unmanaged AM
        appContext.setUnmanagedAM(true);
        LOG.info("Setting unmanaged AM");

        // Submit the application to the applications manager
        LOG.info("Submitting application to ASM");
        rmClient.submitApplication(appContext);

        ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
                YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));

        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            // Monitor the application attempt to wait for launch state
            ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId,
                    YarnApplicationAttemptState.LAUNCHED);
            ApplicationAttemptId attemptId = attemptReport.getApplicationAttemptId();
            LOG.info("Launching AM with application attempt id " + attemptId);
            // launch AM
            launchAM(attemptId);
            // Monitor the application for end state
            appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
                    YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        }

        YarnApplicationState appState = appReport.getYarnApplicationState();
        FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

        LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);

        boolean success;
        if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
            LOG.info("Application has completed successfully.");
            success = true;
        } else {
            LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString()
                    + ", FinalStatus=" + appStatus.toString());
            success = false;
        }

        return success;
    } finally {
        rmClient.stop();
    }
}

---

From source file:alluxio.yarn.ApplicationMaster.java

License:Apache License

/**
 * Submits requests for containers until the master and all workers are launched.
 *
 * @throws Exception if an error occurs while requesting or launching containers
 *//* ww  w  .  jav  a2s  .c  o  m*/
public void requestAndLaunchContainers() throws Exception {
    Resource masterResource = Records.newRecord(Resource.class);
    masterResource.setMemory(mMasterMemInMB);
    masterResource.setVirtualCores(mMasterCpu);
    mContainerAllocator = new ContainerAllocator("master", 1, 1, masterResource, mYarnClient, mRMClient,
            mMasterAddress);
    List<Container> masterContainers = mContainerAllocator.allocateContainers();
    launchMasterContainer(Iterables.getOnlyElement(masterContainers));

    Resource workerResource = Records.newRecord(Resource.class);
    workerResource.setMemory(mWorkerMemInMB + mRamdiskMemInMB);
    workerResource.setVirtualCores(mWorkerCpu);
    mContainerAllocator = new ContainerAllocator("worker", mNumWorkers, mMaxWorkersPerHost, workerResource,
            mYarnClient, mRMClient);
    List<Container> workerContainers = mContainerAllocator.allocateContainers();
    for (Container container : workerContainers) {
        launchWorkerContainer(container);
    }
    LOG.info("Master and workers are launched");
}

---

From source file:alluxio.yarn.ApplicationMaster.java

License:Apache License

private void launchMasterContainer(Container container) {
    String command = YarnUtils.buildCommand(YarnContainerType.ALLUXIO_MASTER);
    try {//from  ww w.  ja v  a 2s  . c  o  m
        ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
        ctx.setCommands(Lists.newArrayList(command));
        ctx.setLocalResources(setupLocalResources(mResourcePath));
        ctx.setEnvironment(setupMasterEnvironment());
        if (UserGroupInformation.isSecurityEnabled()) {
            ctx.setTokens(mAllTokens.duplicate());
        }
        LOG.info("Launching container {} for Alluxio master on {} with master command: {}", container.getId(),
                container.getNodeHttpAddress(), command);
        mNMClient.startContainer(container, ctx);
        String containerUri = container.getNodeHttpAddress(); // in the form of 1.2.3.4:8042
        mMasterContainerNetAddress = containerUri.split(":")[0];
        LOG.info("Master address: {}", mMasterContainerNetAddress);
        return;
    } catch (Exception e) {
        LOG.error("Error launching container {}", container.getId(), e);
    }
}

---

From source file:alluxio.yarn.ApplicationMaster.java

License:Apache License

private void launchWorkerContainer(Container container) {
    String command = YarnUtils.buildCommand(YarnContainerType.ALLUXIO_WORKER);

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    ctx.setCommands(Lists.newArrayList(command));
    ctx.setLocalResources(setupLocalResources(mResourcePath));
    ctx.setEnvironment(setupWorkerEnvironment(mMasterContainerNetAddress, mRamdiskMemInMB));
    if (UserGroupInformation.isSecurityEnabled()) {
        ctx.setTokens(mAllTokens.duplicate());
    }//from w  ww. j a  v a2  s . c o m

    try {
        LOG.info("Launching container {} for Alluxio worker on {} with worker command: {}", container.getId(),
                container.getNodeHttpAddress(), command);
        mNMClient.startContainer(container, ctx);
    } catch (Exception e) {
        LOG.error("Error launching container {}", container.getId(), e);
    }
}

---

From source file:alluxio.yarn.ApplicationMasterTest.java

License:Apache License

private static Map<String, LocalResource> getExpectedLocalResources() {
    Map<String, LocalResource> resources = Maps.newHashMap();
    resources.put(YarnUtils.ALLUXIO_TARBALL, Records.newRecord(LocalResource.class));
    resources.put(YarnUtils.ALLUXIO_SETUP_SCRIPT, Records.newRecord(LocalResource.class));
    return resources;
}

---

From source file:alluxio.yarn.Client.java

License:Apache License

/**
 * Submits an application to the ResourceManager to run ApplicationMaster.
 *
 * The stable Yarn API provides a convenience method (YarnClient#createApplication) for creating
 * applications and setting up the application submission context. This was not available in the
 * alpha API./*from  w w  w.ja  v  a2 s. c om*/
 */
private void submitApplication() throws YarnException, IOException {
    // Initialize a YarnClient
    mYarnClient = YarnClient.createYarnClient();
    mYarnClient.init(mYarnConf);
    mYarnClient.start();

    // Create an application, get and check the information about the cluster
    YarnClientApplication app = mYarnClient.createApplication();
    // Get a response of this application, containing information of the cluster
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // Check if the cluster has enough resource to launch the ApplicationMaster
    checkClusterResource(appResponse);

    // Check that there are enough hosts in the cluster to support the desired number of workers
    checkNodesAvailable();

    // Set up the container launch context for the application master
    mAmContainer = Records.newRecord(ContainerLaunchContext.class);
    setupContainerLaunchContext();

    // Finally, set-up ApplicationSubmissionContext for the application
    mAppContext = app.getApplicationSubmissionContext();
    setupApplicationSubmissionContext();

    // Submit the application to the applications manager.
    // Ignore the response as either a valid response object is returned on success
    // or an exception thrown to denote some form of a failure
    mAppId = mAppContext.getApplicationId();
    System.out.println("Submitting application of id " + mAppId + " to ResourceManager");
    mYarnClient.submitApplication(mAppContext);
    monitorApplication();
}

---

From source file:alluxio.yarn.ContainerAllocatorTest.java

License:Apache License

private ContainerAllocator setup(int numHosts, int maxContainersPerHost, int numContainers) throws Exception {
    ContainerAllocator containerAllocator = new ContainerAllocator(CONTAINER_NAME, numContainers,
            maxContainersPerHost, mResource, mYarnClient, mRMClient);
    List<NodeReport> nodeReports = new ArrayList<>();
    for (int i = 0; i < numHosts; i++) {
        NodeReport nodeReport = Records.newRecord(NodeReport.class);
        nodeReport.setNodeId(NodeId.newInstance("host" + i, 0));
        nodeReports.add(nodeReport);/*from w  w  w  .j  a va 2 s  .  co m*/
    }
    when(mYarnClient.getNodeReports(Matchers.<NodeState[]>anyVararg())).thenReturn(nodeReports);
    doAnswer(allocateFirstHostAnswer(containerAllocator)).when(mRMClient)
            .addContainerRequest(any(ContainerRequest.class));
    return containerAllocator;
}

---

From source file:alluxio.yarn.ContainerAllocatorTest.java

License:Apache License

private Answer<Void> allocateFirstHostAnswer(final ContainerAllocator containerAllocator) {
    return new Answer<Void>() {
        @Override//from   w  ww.  ja  v a 2  s.com
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ContainerRequest containerRequest = invocation.getArgumentAt(0, ContainerRequest.class);
            Container container = Records.newRecord(Container.class);
            container.setNodeId(NodeId.newInstance(containerRequest.getNodes().get(0), 0));
            containerAllocator.allocateContainer(container);
            return null;
        }
    };
}

---

From source file:alluxio.yarn.YarnUtils.java

License:Apache License

/**
 * Creates a local resource for a file on HDFS.
 *
 * @param yarnConf YARN configuration//  w  w w .j  a  v  a 2 s .c  o  m
 * @param resource the path to a resource file on HDFS
 * @throws IOException if the file can not be found on HDFS
 * @return the created local resource
 */
public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resource)
        throws IOException {
    LocalResource localResource = Records.newRecord(LocalResource.class);

    Path resourcePath = new Path(resource);

    FileStatus jarStat = FileSystem.get(resourcePath.toUri(), yarnConf).getFileStatus(resourcePath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
    return localResource;
}

---

From source file:azkaban.security.HadoopSecurityManager_H_2_0.java

License:Apache License

private void cancelJhsToken(final Token<? extends TokenIdentifier> t, String userToProxy)
        throws HadoopSecurityManagerException {
    // it appears yarn would clean up this token after app finish, after a long
    // while though.
    org.apache.hadoop.yarn.api.records.Token token = org.apache.hadoop.yarn.api.records.Token
            .newInstance(t.getIdentifier(), t.getKind().toString(), t.getPassword(), t.getService().toString());
    final YarnRPC rpc = YarnRPC.create(conf);
    final InetSocketAddress jhsAddress = SecurityUtil.getTokenServiceAddr(t);
    MRClientProtocol jhsProxy = null;/*from   w  ww . ja  va 2s. c om*/
    try {
        jhsProxy = UserGroupInformation.getCurrentUser().doAs(new PrivilegedAction<MRClientProtocol>() {
            @Override
            public MRClientProtocol run() {
                return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, jhsAddress, conf);
            }
        });
        CancelDelegationTokenRequest request = Records.newRecord(CancelDelegationTokenRequest.class);
        request.setDelegationToken(token);
        jhsProxy.cancelDelegationToken(request);
    } catch (Exception e) {
        throw new HadoopSecurityManagerException("Failed to cancel token. " + e.getMessage() + e.getCause(), e);
    } finally {
        RPC.stopProxy(jhsProxy);
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »