Example usage for org.apache.hadoop.yarn.client.api AMRMClient createAMRMClient

List of usage examples for org.apache.hadoop.yarn.client.api AMRMClient createAMRMClient

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.client.api.*
  5. AMRMClient
  6. createAMRMClient

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api AMRMClient createAMRMClient.

Prototype

@Public
public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient()

Source Link

Document

Create a new instance of AMRMClient.

Usage

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void _registerSchedulerAndCreateNMClient(String queue) throws Exception {
    NMTokenCache nmTokenCache = new NMTokenCache();
    nmClient = NMClient.createNMClient();
    nmClient.setNMTokenCache(nmTokenCache);
    nmClient.init(yarnConf);/*ww w  .  j  a  v a  2s.  c o m*/
    nmClient.start();
    LOG.debug("Started NMClient, AM '{}' with scheduler for '{}' queue", appId, queue);
    int heartbeatInterval = getConf().getInt(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERNAL_DEFAULT);
    AMRMClient<LlamaContainerRequest> amRmClient = AMRMClient.createAMRMClient();
    amRmClient.setNMTokenCache(nmTokenCache);
    amRmClientAsync = AMRMClientAsync.createAMRMClientAsync(amRmClient, heartbeatInterval,
            YarnRMConnector.this);
    amRmClientAsync.init(yarnConf);
    amRmClientAsync.start();
    String urlWithoutScheme = getConf().get(ADVERTISED_TRACKING_URL_KEY, "http://")
            .substring("http://".length());
    RegisterApplicationMasterResponse response = amRmClientAsync.registerApplicationMaster(
            getConf().get(ADVERTISED_HOSTNAME_KEY, ""), getConf().getInt(ADVERTISED_PORT_KEY, 0),
            urlWithoutScheme);
    maxResource = response.getMaximumResourceCapability();
    nodes = Collections.synchronizedMap(new HashMap<String, Resource>());
    for (NodeReport nodeReport : yarnClient.getNodeReports()) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        }
    }
    LOG.debug("Registered with scheduler, AM '{}' for '{}' queue", appId, queue);
}

From source file:com.continuuity.weave.internal.yarn.Hadoop21YarnAMClient.java

License:Apache License

public Hadoop21YarnAMClient(Configuration conf) {
    String masterContainerId = System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.name());
    Preconditions.checkArgument(masterContainerId != null, "Missing %s from environment",
            ApplicationConstants.Environment.CONTAINER_ID.name());
    this.containerId = ConverterUtils.toContainerId(masterContainerId);
    this.containerRequests = ArrayListMultimap.create();

    this.amrmClient = AMRMClient.createAMRMClient();
    this.amrmClient.init(conf);
    this.nmClient = new Hadoop21YarnNMClient(conf);
}

From source file:com.datatorrent.stram.StramMiniClusterTest.java

License:Apache License

@Ignore
@Test//from w  w  w. ja  va 2  s. c o  m
public void testUnmanagedAM2() throws Exception {

    new InlineAM(conf) {

        @Override
        @SuppressWarnings("SleepWhileInLoop")
        public void runAM(ApplicationAttemptId attemptId) throws Exception {
            LOG.debug("AM running {}", attemptId);
            AMRMClient<ContainerRequest> amRmClient = AMRMClient.createAMRMClient();
            amRmClient.init(conf);
            amRmClient.start();

            // register with the RM (LAUNCHED -> RUNNING)
            amRmClient.registerApplicationMaster("", 0, null);

            // AM specific logic

            String[] hosts = { "vm1" };
            String[] racks = { "somerack" };

            Resource capability = Records.newRecord(Resource.class);
            capability.setMemory(1000);

            Priority priority = Records.newRecord(Priority.class);
            priority.setPriority(10);
            AMRMClient.ContainerRequest req = new AMRMClient.ContainerRequest(capability, hosts, racks,
                    priority);
            amRmClient.addContainerRequest(req);
            amRmClient.addContainerRequest(req);

            /*
                    capability = Records.newRecord(Resource.class);
                    capability.setMemory(5512);
                    priority = Records.newRecord(Priority.class);
                    priority.setPriority(11);
                    req = new AMRMClient.ContainerRequest(capability, hosts, racks, priority, 3);
                    amRmClient.addContainerRequest(req);
            */
            for (int i = 0; i < 100; i++) {
                AllocateResponse ar = amRmClient.allocate(0);
                sleep(1000);
                LOG.debug("allocateResponse: {}", ar);
                for (Container c : ar.getAllocatedContainers()) {
                    LOG.debug("*** allocated {}", c.getResource());
                    amRmClient.removeContainerRequest(req);
                }
                /*
                GetClusterNodesRequest request =
                    Records.newRecord(GetClusterNodesRequest.class);
                ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
                GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
                List<NodeReport> nodeReports = response.getNodeReports();
                LOG.info(nodeReports);
                        
                for (NodeReport nr: nodeReports) {
                  LOG.info("Node: " + nr.getNodeId());
                  LOG.info("Total memory: " + nr.getCapability());
                  LOG.info("Used memory: " + nr.getUsed());
                  LOG.info("Number containers: " + nr.getNumContainers());
                }
                */
            }

            // unregister from RM
            amRmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "testUnmanagedAM finished",
                    null);

        }

    }.run();

}

From source file:com.datatorrent.stram.StreamingAppMasterService.java

License:Apache License

@Override
protected void serviceInit(Configuration conf) throws Exception {
    LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp="
            + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId="
            + appAttemptID.getAttemptId());

    FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
    try {//from   w w w  .j  a v  a  2s  . com
        this.dag = LogicalPlan.read(fis);
    } finally {
        fis.close();
    }
    // "debug" simply dumps all data using LOG.info
    if (dag.isDebug()) {
        dumpOutDebugInfo();
    }

    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
    this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
    dag = this.dnmgr.getLogicalPlan();
    this.appContext = new ClusterAppContextImpl(dag.getAttributes());

    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
    StringCodecs.loadConverters(codecs);

    LOG.info("Starting application with {} operators in {} containers",
            dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

    if (UserGroupInformation.isSecurityEnabled()) {
        // TODO :- Need to perform token renewal
        delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL,
                DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL,
                DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
    }
    this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
    addService(nmClient);
    this.amRmClient = AMRMClient.createAMRMClient();
    addService(amRmClient);

    // start RPC server
    int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
    this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr,
            delegationTokenManager, rpcListenerCount);
    addService(heartbeatListener);

    String appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
    if (appDataPushTransport != null) {
        this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
        addService(this.appDataPushAgent);
    }
    // initialize all services added above
    super.serviceInit(conf);
}

From source file:com.ibm.bi.dml.yarn.DMLAppMaster.java

License:Open Source License

/**
 * /*  ww  w  . j  av a  2s .  c om*/
 * @param args
 * @throws YarnException 
 * @throws IOException 
 */
public void runApplicationMaster(String[] args) throws YarnException, IOException {
    _conf = new YarnConfiguration();

    //obtain application ID
    String containerIdString = System.getenv(Environment.CONTAINER_ID.name());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    _appId = containerId.getApplicationAttemptId().getApplicationId();
    LOG.info("SystemML appplication master (applicationID: " + _appId + ")");

    //initialize clients to ResourceManager
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(_conf);
    rmClient.start();

    //register with ResourceManager
    rmClient.registerApplicationMaster("", 0, ""); //host, port for rm communication
    LOG.debug("Registered the SystemML application master with resource manager");

    //start status reporter to ResourceManager
    DMLAppMasterStatusReporter reporter = new DMLAppMasterStatusReporter(rmClient, 10000);
    reporter.start();
    LOG.debug("Started status reporter (heartbeat to resource manager)");

    //set DMLscript app master context
    DMLScript.setActiveAM();

    //parse input arguments
    String[] otherArgs = new GenericOptionsParser(_conf, args).getRemainingArgs();

    //run SystemML CP
    FinalApplicationStatus status = null;
    try {
        //core dml script execution (equivalent to non-AM runtime)
        boolean success = DMLScript.executeScript(_conf, otherArgs);

        if (success)
            status = FinalApplicationStatus.SUCCEEDED;
        else
            status = FinalApplicationStatus.FAILED;
    } catch (DMLScriptException ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script due to stop call:\n\t"
                + ex.getMessage());
        status = FinalApplicationStatus.FAILED;
        writeMessageToHDFSWorkingDir(ex.getMessage());
    } catch (Exception ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script.", ex);
        status = FinalApplicationStatus.FAILED;
    } finally {
        //stop periodic status reports
        reporter.stopStatusReporter();
        LOG.debug("Stopped status reporter");

        //unregister resource manager client
        rmClient.unregisterApplicationMaster(status, "", "");
        LOG.debug("Unregistered the SystemML application master");
    }
}

From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java

License:Apache License

/**
 * Main run function for the application master
 *
 * @throws YarnException/*from w  w  w.j  a v  a 2 s. co  m*/
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
    yarnClient.start();
    LOG.info("Starting ApplicationMaster");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    amRMClient = AMRMClient.createAMRMClient();
    amRMClient.init(conf);
    amRMClient.start();

    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();

    // Register self with ResourceManager
    // This will start heartbeating to the RM
    appMasterHostname = NetUtils.getHostname();
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
            appMasterRpcPort, appMasterTrackingUrl);

    //TODO: Figure out how to do this.
    List<NodeReport> reports = this.yarnClient.getNodeReports();
    LOG.info("Cluster Status");
    List<Resource> availableResources = new ArrayList<Resource>();
    for (NodeReport nr : reports) {
        LOG.info("    NodeId: " + nr.getNodeId() + " Capabilities " + nr.getCapability() + " Used Resources "
                + nr.getUsed());
        int availableMem = nr.getCapability().getMemory() - nr.getUsed().getMemory();
        int availableVCores = nr.getCapability().getVirtualCores() - nr.getUsed().getVirtualCores();
        Resource resource = Resource.newInstance(availableMem, availableVCores);

        availableResources.add(resource);
    }

    /*Based on available resources scheduler should decide the best allocation of recommenders to resources
    and return a list of resources that should be requested from the ResourceManager*/
    DataSetDescription datasetDesc = new DataSetDescription(this.setup.dataMetadataFile);
    List<RecommenderPool> recommenderPools = RecommenderScheduler.splitRecommenderPool(availableResources,
            recommenders, datasetDesc, this.setup.nShards);

    for (RecommenderPool res : recommenderPools) {
        ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority);
        LOG.info("CONTAINER ASK: " + containerAsk);
        amRMClient.addContainerRequest(containerAsk);
    }

    float progress = 0;

    List<RecommenderPool> pendingPools = new ArrayList<RecommenderPool>();
    for (RecommenderPool p : recommenderPools)
        pendingPools.add(p);

    this.numTotalContainers = recommenderPools.size();
    this.numCompletedContainers.set(0);
    this.numAllocatedContainers.set(0);

    while (numCompletedContainers.get() != numTotalContainers) {
        try {
            Thread.sleep(200);

            AllocateResponse allocResp = amRMClient.allocate(progress);
            List<Container> newContainers = allocResp.getAllocatedContainers();
            List<ContainerStatus> completedContainers = allocResp.getCompletedContainersStatuses();

            if (this.numAllocatedContainers.get() >= this.numTotalContainers && pendingPools.size() != 0) {
                //Ask for new containers for pending pools
                LOG.warn("The number of allocated containers has exceeded number of total containers, but "
                        + "the pending pool size is still not 0. Asking for new containers from RM");
                for (RecommenderPool res : pendingPools) {
                    ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(),
                            requestPriority);
                    LOG.info("NEW CONTAINER ASK: " + containerAsk);
                    amRMClient.addContainerRequest(containerAsk);
                }

            }

            if (newContainers.size() > 0) {
                LOG.info("Allocated " + newContainers.size() + " new containers");
                numAllocatedContainers.addAndGet(newContainers.size());
                for (Container container : newContainers) {
                    //Find matching recommender pool from pendingRecommender pools.
                    RecommenderPool pool = null;
                    for (RecommenderPool p : pendingPools) {
                        if (p.getTotalMemory() == container.getResource().getMemory()) {
                            pool = p;
                            break;
                        }
                    }
                    if (pool == null) {
                        LOG.warn("No Takers for Container " + container + " Releasing container");
                        amRMClient.releaseAssignedContainer(container.getId());
                    } else {
                        startContainer(container, pool);
                        //This pool has now got a container. Remove it from pending pools
                        pendingPools.remove(pool);
                    }
                }

            }

            onContainersCompleted(completedContainers);

        } catch (InterruptedException ex) {
        }
    }
    finish();

    return success;
}

From source file:eu.stratosphere.yarn.ApplicationMaster.java

License:Apache License

private void run() throws Exception {
    //Utils.logFilesInCurrentDirectory(LOG);
    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = Utils.initializeYarnConfiguration();
    FileSystem fs = FileSystem.get(conf);
    Map<String, String> envs = System.getenv();
    final String currDir = envs.get(Environment.PWD.key());
    final String logDirs = envs.get(Environment.LOG_DIRS.key());
    final String ownHostname = envs.get(Environment.NM_HOST.key());
    final String appId = envs.get(Client.ENV_APP_ID);
    final String clientHomeDir = envs.get(Client.ENV_CLIENT_HOME_DIR);
    final String applicationMasterHost = envs.get(Environment.NM_HOST.key());
    final String remoteStratosphereJarPath = envs.get(Client.STRATOSPHERE_JAR_PATH);
    final String shipListString = envs.get(Client.ENV_CLIENT_SHIP_FILES);
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final int taskManagerCount = Integer.valueOf(envs.get(Client.ENV_TM_COUNT));
    final int memoryPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_MEMORY));
    final int coresPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_CORES));

    int heapLimit = Utils.calculateHeapSize(memoryPerTaskManager);

    if (currDir == null) {
        throw new RuntimeException("Current directory unknown");
    }/*from  w  w  w  .jav a 2  s . c  o  m*/
    if (ownHostname == null) {
        throw new RuntimeException("Own hostname (" + Environment.NM_HOST + ") not set.");
    }
    LOG.info("Working directory " + currDir);

    // load Stratosphere configuration.
    Utils.getStratosphereConfiguration(currDir);

    final String localWebInterfaceDir = currDir + "/resources/"
            + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME;

    // Update yaml conf -> set jobManager address to this machine's address.
    FileInputStream fis = new FileInputStream(currDir + "/stratosphere-conf.yaml");
    BufferedReader br = new BufferedReader(new InputStreamReader(fis));
    Writer output = new BufferedWriter(new FileWriter(currDir + "/stratosphere-conf-modified.yaml"));
    String line;
    while ((line = br.readLine()) != null) {
        if (line.contains(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
        } else if (line.contains(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + "\n");
        } else {
            output.append(line + "\n");
        }
    }
    // just to make sure.
    output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + localWebInterfaceDir + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY + ": " + logDirs + "\n");
    output.close();
    br.close();
    File newConf = new File(currDir + "/stratosphere-conf-modified.yaml");
    if (!newConf.exists()) {
        LOG.warn("modified yaml does not exist!");
    }

    Utils.copyJarContents("resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME,
            ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath());

    JobManager jm;
    {
        String pathToNepheleConfig = currDir + "/stratosphere-conf-modified.yaml";
        String[] args = { "-executionMode", "cluster", "-configDir", pathToNepheleConfig };

        // start the job manager
        jm = JobManager.initialize(args);

        // Start info server for jobmanager
        jm.startInfoServer();
    }

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    LOG.info("registering ApplicationMaster");
    rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://" + applicationMasterHost + ":"
            + GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined"));

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memoryPerTaskManager);
    capability.setVirtualCores(coresPerTaskManager);

    // Make container requests to ResourceManager
    for (int i = 0; i < taskManagerCount; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        LOG.info("Requesting TaskManager container " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    LocalResource stratosphereJar = Records.newRecord(LocalResource.class);
    LocalResource stratosphereConf = Records.newRecord(LocalResource.class);

    // register Stratosphere Jar with remote HDFS
    final Path remoteJarPath = new Path(remoteStratosphereJarPath);
    Utils.registerLocalResource(fs, remoteJarPath, stratosphereJar);

    // register conf with local fs.
    Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId,
            new Path("file://" + currDir + "/stratosphere-conf-modified.yaml"), stratosphereConf,
            new Path(clientHomeDir));
    LOG.info("Prepared localresource for modified yaml: " + stratosphereConf);

    boolean hasLog4j = new File(currDir + "/log4j.properties").exists();
    // prepare the files to ship
    LocalResource[] remoteShipRsc = null;
    String[] remoteShipPaths = shipListString.split(",");
    if (!shipListString.isEmpty()) {
        remoteShipRsc = new LocalResource[remoteShipPaths.length];
        { // scope for i
            int i = 0;
            for (String remoteShipPathStr : remoteShipPaths) {
                if (remoteShipPathStr == null || remoteShipPathStr.isEmpty()) {
                    continue;
                }
                remoteShipRsc[i] = Records.newRecord(LocalResource.class);
                Path remoteShipPath = new Path(remoteShipPathStr);
                Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]);
                i++;
            }
        }
    }

    // respect custom JVM options in the YAML file
    final String javaOpts = GlobalConfiguration.getString(ConfigConstants.STRATOSPHERE_JVM_OPTIONS, "");

    // Obtain allocated containers and launch
    int allocatedContainers = 0;
    int completedContainers = 0;
    while (allocatedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(0);
        for (Container container : response.getAllocatedContainers()) {
            LOG.info("Got new Container for TM " + container.getId() + " on host "
                    + container.getNodeId().getHost());
            ++allocatedContainers;

            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

            String tmCommand = "$JAVA_HOME/bin/java -Xmx" + heapLimit + "m " + javaOpts;
            if (hasLog4j) {
                tmCommand += " -Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                        + "/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties";
            }
            tmCommand += " eu.stratosphere.yarn.YarnTaskManagerRunner -configDir . " + " 1>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stdout.log" + " 2>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stderr.log";
            ctx.setCommands(Collections.singletonList(tmCommand));

            LOG.info("Starting TM with command=" + tmCommand);

            // copy resources to the TaskManagers.
            Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2);
            localResources.put("stratosphere.jar", stratosphereJar);
            localResources.put("stratosphere-conf.yaml", stratosphereConf);

            // add ship resources
            if (!shipListString.isEmpty()) {
                Preconditions.checkNotNull(remoteShipRsc);
                for (int i = 0; i < remoteShipPaths.length; i++) {
                    localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]);
                }
            }

            ctx.setLocalResources(localResources);

            // Setup CLASSPATH for Container (=TaskTracker)
            Map<String, String> containerEnv = new HashMap<String, String>();
            Utils.setupEnv(conf, containerEnv); //add stratosphere.jar to class path.
            containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername);

            ctx.setEnvironment(containerEnv);

            UserGroupInformation user = UserGroupInformation.getCurrentUser();
            try {
                Credentials credentials = user.getCredentials();
                DataOutputBuffer dob = new DataOutputBuffer();
                credentials.writeTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
                ctx.setTokens(securityTokens);
            } catch (IOException e) {
                LOG.warn("Getting current user info failed when trying to launch the container"
                        + e.getMessage());
            }

            LOG.info("Launching container " + allocatedContainers);
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container (while allocating) " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(100);
    }

    // Now wait for containers to complete

    while (completedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(completedContainers / taskManagerCount);
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(5000);
    }
    LOG.info("Shutting down JobManager");
    jm.shutdown();

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");

}

From source file:org.apache.flink.yarn.ApplicationMaster.java

License:Apache License

private void run() throws Exception {
    //Utils.logFilesInCurrentDirectory(LOG);
    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = Utils.initializeYarnConfiguration();
    FileSystem fs = FileSystem.get(conf);
    Map<String, String> envs = System.getenv();
    final String currDir = envs.get(Environment.PWD.key());
    final String logDirs = envs.get(Environment.LOG_DIRS.key());
    final String ownHostname = envs.get(Environment.NM_HOST.key());
    final String appId = envs.get(Client.ENV_APP_ID);
    final String clientHomeDir = envs.get(Client.ENV_CLIENT_HOME_DIR);
    final String applicationMasterHost = envs.get(Environment.NM_HOST.key());
    final String remoteFlinkJarPath = envs.get(Client.FLINK_JAR_PATH);
    final String shipListString = envs.get(Client.ENV_CLIENT_SHIP_FILES);
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final int taskManagerCount = Integer.valueOf(envs.get(Client.ENV_TM_COUNT));
    final int memoryPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_MEMORY));
    final int coresPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_CORES));

    int heapLimit = Utils.calculateHeapSize(memoryPerTaskManager);

    if (currDir == null) {
        throw new RuntimeException("Current directory unknown");
    }//from   w  w  w  .  j  a v a 2s. c om
    if (ownHostname == null) {
        throw new RuntimeException("Own hostname (" + Environment.NM_HOST + ") not set.");
    }
    LOG.info("Working directory " + currDir);

    // load Flink configuration.
    Utils.getFlinkConfiguration(currDir);

    final String localWebInterfaceDir = currDir + "/resources/"
            + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME;

    // Update yaml conf -> set jobManager address to this machine's address.
    FileInputStream fis = new FileInputStream(currDir + "/flink-conf.yaml");
    BufferedReader br = new BufferedReader(new InputStreamReader(fis));
    Writer output = new BufferedWriter(new FileWriter(currDir + "/flink-conf-modified.yaml"));
    String line;
    while ((line = br.readLine()) != null) {
        if (line.contains(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
        } else if (line.contains(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + "\n");
        } else {
            output.append(line + "\n");
        }
    }
    // just to make sure.
    output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + localWebInterfaceDir + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY + ": " + logDirs + "\n");
    output.close();
    br.close();
    File newConf = new File(currDir + "/flink-conf-modified.yaml");
    if (!newConf.exists()) {
        LOG.warn("modified yaml does not exist!");
    }

    Utils.copyJarContents("resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME,
            ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath());

    JobManager jm;
    {
        String pathToNepheleConfig = currDir + "/flink-conf-modified.yaml";
        String[] args = { "-executionMode", "cluster", "-configDir", pathToNepheleConfig };

        // start the job manager
        jm = JobManager.initialize(args);

        // Start info server for jobmanager
        jm.startInfoServer();
    }

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    LOG.info("registering ApplicationMaster");
    rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://" + applicationMasterHost + ":"
            + GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined"));

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memoryPerTaskManager);
    capability.setVirtualCores(coresPerTaskManager);

    // Make container requests to ResourceManager
    for (int i = 0; i < taskManagerCount; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        LOG.info("Requesting TaskManager container " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    LocalResource flinkJar = Records.newRecord(LocalResource.class);
    LocalResource flinkConf = Records.newRecord(LocalResource.class);

    // register Flink Jar with remote HDFS
    final Path remoteJarPath = new Path(remoteFlinkJarPath);
    Utils.registerLocalResource(fs, remoteJarPath, flinkJar);

    // register conf with local fs.
    Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId,
            new Path("file://" + currDir + "/flink-conf-modified.yaml"), flinkConf, new Path(clientHomeDir));
    LOG.info("Prepared localresource for modified yaml: " + flinkConf);

    boolean hasLog4j = new File(currDir + "/log4j.properties").exists();
    // prepare the files to ship
    LocalResource[] remoteShipRsc = null;
    String[] remoteShipPaths = shipListString.split(",");
    if (!shipListString.isEmpty()) {
        remoteShipRsc = new LocalResource[remoteShipPaths.length];
        { // scope for i
            int i = 0;
            for (String remoteShipPathStr : remoteShipPaths) {
                if (remoteShipPathStr == null || remoteShipPathStr.isEmpty()) {
                    continue;
                }
                remoteShipRsc[i] = Records.newRecord(LocalResource.class);
                Path remoteShipPath = new Path(remoteShipPathStr);
                Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]);
                i++;
            }
        }
    }

    // respect custom JVM options in the YAML file
    final String javaOpts = GlobalConfiguration.getString(ConfigConstants.FLINK_JVM_OPTIONS, "");

    // Obtain allocated containers and launch
    int allocatedContainers = 0;
    int completedContainers = 0;
    while (allocatedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(0);
        for (Container container : response.getAllocatedContainers()) {
            LOG.info("Got new Container for TM " + container.getId() + " on host "
                    + container.getNodeId().getHost());
            ++allocatedContainers;

            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

            String tmCommand = "$JAVA_HOME/bin/java -Xmx" + heapLimit + "m " + javaOpts;
            if (hasLog4j) {
                tmCommand += " -Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                        + "/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties";
            }
            tmCommand += " org.apache.flink.yarn.YarnTaskManagerRunner -configDir . " + " 1>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stdout.log" + " 2>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stderr.log";
            ctx.setCommands(Collections.singletonList(tmCommand));

            LOG.info("Starting TM with command=" + tmCommand);

            // copy resources to the TaskManagers.
            Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2);
            localResources.put("flink.jar", flinkJar);
            localResources.put("flink-conf.yaml", flinkConf);

            // add ship resources
            if (!shipListString.isEmpty()) {
                Preconditions.checkNotNull(remoteShipRsc);
                for (int i = 0; i < remoteShipPaths.length; i++) {
                    localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]);
                }
            }

            ctx.setLocalResources(localResources);

            // Setup CLASSPATH for Container (=TaskTracker)
            Map<String, String> containerEnv = new HashMap<String, String>();
            Utils.setupEnv(conf, containerEnv); //add flink.jar to class path.
            containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername);

            ctx.setEnvironment(containerEnv);

            UserGroupInformation user = UserGroupInformation.getCurrentUser();
            try {
                Credentials credentials = user.getCredentials();
                DataOutputBuffer dob = new DataOutputBuffer();
                credentials.writeTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
                ctx.setTokens(securityTokens);
            } catch (IOException e) {
                LOG.warn("Getting current user info failed when trying to launch the container"
                        + e.getMessage());
            }

            LOG.info("Launching container " + allocatedContainers);
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container (while allocating) " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(100);
    }

    // Now wait for containers to complete

    while (completedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(completedContainers / taskManagerCount);
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(5000);
    }
    LOG.info("Shutting down JobManager");
    jm.shutdown();

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");

}

From source file:org.apache.flink.yarn.appMaster.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    // execute Application Master using the client's user
    final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink ApplicationMaster/JobManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);// w w  w . j  a v  a  2s .c om
    }
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            AMRMClient<ContainerRequest> rmClient = null;
            ApplicationMaster am = null;
            try {
                Configuration conf = Utils.initializeYarnConfiguration();
                rmClient = AMRMClient.createAMRMClient();
                rmClient.init(conf);
                rmClient.start();

                // run the actual Application Master
                am = new ApplicationMaster(conf);
                am.generateConfigurationFile();
                am.startJobManager();
                am.setRMClient(rmClient);
                am.run();
            } catch (Throwable e) {
                LOG.error("Error while running the application master", e);
                // the AM is not available. Report error through the unregister function.
                if (rmClient != null && am == null) {
                    try {
                        rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED,
                                "Flink YARN Application master" + " stopped unexpectedly with an exception.\n"
                                        + StringUtils.stringifyException(e),
                                "");
                    } catch (Exception e1) {
                        LOG.error("Unable to fail the application master", e1);
                    }
                    LOG.info("AM unregistered from RM");
                    return null;
                }
                if (rmClient == null) {
                    LOG.error("Unable to unregister AM since the RM client is not available");
                }
                if (am != null) {
                    LOG.info("Writing error into internal message system");
                    am.setFailed(true);
                    am.addMessage(new Message("The application master failed with an exception:\n"
                            + StringUtils.stringifyException(e)));
                    am.keepRPCAlive();
                }
            }
            return null;
        }
    });
}

From source file:org.apache.oozie.action.hadoop.AMRMClientAsyncFactory.java

License:Apache License

public AMRMClientAsync<?> createAMRMClientAsync(int intervalMs, AMRMCallBackHandler callBackHandler) {
    AMRMClient<?> amRmClient = AMRMClient.createAMRMClient();
    AMRMClientAsync<?> amRmClientAsync = AMRMClientAsync.createAMRMClientAsync(amRmClient, intervalMs,
            callBackHandler);/* w  w  w.  jav  a2  s . c  om*/
    return amRmClientAsync;
}