Example usage for org.apache.hadoop.yarn.api.records ApplicationAttemptId getAttemptId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationAttemptId getAttemptId.

Prototype

@Public
@Stable
public abstract int getAttemptId();

Source Link

Document

Get the attempt id of the Application.

Usage

From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java

License:Apache License

/** Parse command line arguments, initialize HDFS, manage environment variables. */
public boolean init(String[] args) throws ParseException, IOException, JSONException {

    DefaultMetricsSystem.initialize("ApplicationMaster");

    Options opts = new Options();
    opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");
    opts.addOption("u", "summary", true,
            "The name of the json summary file. No file is created if this parameter is not specified.");
    opts.addOption("m", "memory", true,
            "The amount of memory (in MB) to be allocated per worker container. Overrides settings in hiway-site.xml.");
    opts.addOption("c", "custom", true,
            "The name of an (optional) JSON file, in which custom amounts of memory can be specified per task.");
    opts.addOption("s", "scheduler", true,
            "The scheduling policy that is to be employed. Valid arguments: "
                    + Arrays.toString(HiWayConfiguration.HIWAY_SCHEDULERS.values())
                    + ". Overrides settings in hiway-site.xml.");
    opts.addOption("d", "debug", false, "Provide additional logs and information for debugging");
    opts.addOption("v", "verbose", false, "Increase verbosity of output / reporting.");
    opts.addOption("appid", true, "Id of this Application Master.");

    opts.addOption("h", "help", false, "Print usage");
    CommandLine cliParser = new GnuParser().parse(opts, args);

    if (args.length == 0) {
        Logger.printUsage(opts);// ww  w . j a  va2 s . c  o m
        throw new IllegalArgumentException("No args specified for application master to initialize");
    }

    if (cliParser.getArgs().length == 0) {
        Logger.printUsage(opts);
        throw new IllegalArgumentException("No workflow file specified.");
    }

    if (!cliParser.hasOption("appid")) {
        throw new IllegalArgumentException("No id of Application Master specified");
    }

    if (cliParser.hasOption("verbose")) {
        HiWayConfiguration.verbose = true;
    }

    appId = cliParser.getOptionValue("appid");
    try {
        logger.statLog = new BufferedWriter(new FileWriter(appId + ".log"));
    } catch (IOException e) {
        e.printStackTrace(System.out);
        System.exit(-1);
    }

    if (cliParser.hasOption("help")) {
        Logger.printUsage(opts);
        return false;
    }

    if (cliParser.hasOption("debug")) {
        Logger.dumpOutDebugInfo();
        HiWayConfiguration.debug = true;
    }

    if (cliParser.hasOption("summary")) {
        summaryPath = new Path(cliParser.getOptionValue("summary"));
    }

    String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE,
            HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT);
    String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE,
            HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT);
    Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName);
    Data.setHdfsBaseDirectory(hdfsBaseDirectory);
    Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName);
    hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId);
    Data.setHdfsApplicationDirectory(hdfsApplicationDirectory);
    Data.setHdfs(hdfs);

    if (cliParser.hasOption("custom")) {
        Data customMemPath = new Data(cliParser.getOptionValue("custom"));
        customMemPath.stageIn();
        StringBuilder sb = new StringBuilder();
        try (BufferedReader in = new BufferedReader(new FileReader(customMemPath.getLocalPath().toString()))) {
            String line;
            while ((line = in.readLine()) != null) {
                sb.append(line);
            }
        }
        JSONObject obj = new JSONObject(sb.toString());
        Iterator<?> keys = obj.keys();
        while (keys.hasNext()) {
            String key = (String) keys.next();
            int minMem = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
                    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
            int desiredMem = obj.getInt(key);
            customMemoryMap.put(key,
                    (desiredMem % minMem) == 0 ? desiredMem : (desiredMem / minMem + 1) * minMem);
        }
    }

    Map<String, String> envs = System.getenv();

    /* this application's attempt id (combination of attemptId and fail count) */
    ApplicationAttemptId appAttemptID;
    if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
        if (cliParser.hasOption("app_attempt_id")) {
            String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
            appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
        } else {
            throw new IllegalArgumentException("Application Attempt Id not set in the environment");
        }
    } else {
        ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
        appAttemptID = containerId.getApplicationAttemptId();
    }

    if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
        throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HOST.name())) {
        throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
        throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_PORT.name())) {
        throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
    }

    Logger.writeToStdout("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId()
            + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId="
            + appAttemptID.getAttemptId());

    String shellEnvs[] = conf.getStrings(HiWayConfiguration.HIWAY_WORKER_SHELL_ENV,
            HiWayConfiguration.HIWAY_WORKER_SHELL_ENV_DEFAULT);
    for (String env : shellEnvs) {
        env = env.trim();
        int index = env.indexOf('=');
        if (index == -1) {
            shellEnv.put(env, "");
            continue;
        }
        String key = env.substring(0, index);
        String val = "";
        if (index < (env.length() - 1)) {
            val = env.substring(index + 1);
        }
        shellEnv.put(key, val);
    }

    String workflowParam = cliParser.getArgs()[0];
    try {
        workflowPath = new Path(new URI(workflowParam).getPath());
    } catch (URISyntaxException e) {
        workflowPath = new Path(workflowParam);
    }

    schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(conf
            .get(HiWayConfiguration.HIWAY_SCHEDULER, HiWayConfiguration.HIWAY_SCHEDULER_DEFAULT.toString()));
    if (cliParser.hasOption("scheduler")) {
        schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(cliParser.getOptionValue("scheduler"));
    }

    containerMemory = conf.getInt(HiWayConfiguration.HIWAY_WORKER_MEMORY,
            HiWayConfiguration.HIWAY_WORKER_MEMORY_DEFAULT);
    if (cliParser.hasOption("memory")) {
        containerMemory = Integer.parseInt(cliParser.getOptionValue("memory"));
    }

    containerCores = conf.getInt(HiWayConfiguration.HIWAY_WORKER_VCORES,
            HiWayConfiguration.HIWAY_WORKER_VCORES_DEFAULT);
    requestPriority = conf.getInt(HiWayConfiguration.HIWAY_WORKER_PRIORITY,
            HiWayConfiguration.HIWAY_WORKER_PRIORITY_DEFAULT);

    // Create and start the Timeline timelineClient
    if (conf.getBoolean("yarn.timeline-service.enabled", false)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(conf);
        timelineClient.start();
        Logger.writeToStdout("Started TimeLineClient.");
    } else {
        Logger.writeToStdErr("TimeLineClient disabled.");
    }
    return true;
}

---

From source file:org.apache.reef.runtime.yarn.driver.YarnDriverRuntimeRestartManager.java

License:Apache License

/**
 * Determines the number of times the Driver has been submitted based on the container ID environment
 * variable provided by YARN. If that fails, determine whether the application master is a restart
 * based on the number of previous containers reported by YARN. In the failure scenario, returns 1 if restart, 0
 * otherwise.//from   w w w . ja va 2s.co  m
 * @return positive value if the application master is a restarted instance, 0 otherwise.
 */
@Override
public int getResubmissionAttempts() {
    final String containerIdString = getContainerIdString();
    final ApplicationAttemptId appAttemptID = getAppAttemptId(containerIdString);

    if (containerIdString == null || appAttemptID == null) {
        LOG.log(Level.WARNING,
                "Was not able to fetch application attempt, container ID is [" + containerIdString
                        + "] and application attempt is [" + appAttemptID
                        + "]. Determining restart based on previous containers.");

        if (this.isRestartByPreviousContainers()) {
            LOG.log(Level.WARNING,
                    "Driver is a restarted instance based on the number of previous containers. "
                            + "As returned by the Resource Manager. Returning default resubmission attempts "
                            + DEFAULT_RESTART_RESUBMISSION_ATTEMPTS + ".");
            return DEFAULT_RESTART_RESUBMISSION_ATTEMPTS;
        }

        return 0;
    }

    int appAttempt = appAttemptID.getAttemptId();

    LOG.log(Level.FINE, "Application attempt: " + appAttempt);
    assert appAttempt > 0;
    return appAttempt - 1;
}

---

From source file:org.apache.tajo.util.TajoIdUtils.java

License:Apache License

public static QueryId createQueryId(ApplicationAttemptId appAttemptId) {
    QueryId queryId = new QueryId();
    queryId.setApplicationId(appAttemptId.getApplicationId());
    queryId.setAttemptId(appAttemptId.getAttemptId());
    return queryId;
}

---

From source file:org.apache.tajo.yarn.ApplicationMaster.java

License:Apache License

/**
 * Parse command line options//  ww  w  . ja v a2  s  .  co m
 *
 * @param args Command line args
 * @return Whether init successful and getLaunchContext should be invoked
 * @throws org.apache.commons.cli.ParseException
 * @throws java.io.IOException
 */
public boolean init(String[] args) throws ParseException, IOException {

    Options opts = new Options();
    opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");
    opts.addOption("qm_memory", true,
            "Amount of memory in MB to be requested to launch a QueryMaster. Default 512");
    opts.addOption("qm_vcores", true,
            "Amount of virtual cores to be requested to launch a QueryMaster. Default 2");
    opts.addOption("tr_memory", true,
            "Amount of memory in MB to be requested to launch a TaskRunner. Default 1024");
    opts.addOption("tr_vcores", true,
            "Amount of virtual cores to be requested to launch a TaskRunner. Default 4");
    opts.addOption("worker_memory", true,
            "Amount of memory in MB to be requested to launch a worker. Default 2048");
    opts.addOption("worker_vcores", true,
            "Amount of virtual cores to be requested to launch a worker. Default 4");
    opts.addOption("help", false, "Print usage");

    CommandLine cliParser = new GnuParser().parse(opts, args);

    // Check whether customer log4j.properties file exists
    if (fileExist(log4jPath)) {
        try {
            Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
        } catch (Exception e) {
            LOG.warn("Can not set up custom log4j properties. " + e);
        }
    }

    if (cliParser.hasOption("help")) {
        printUsage(opts);
        return false;
    }

    ApplicationAttemptId appAttemptID = null;
    Map<String, String> envs = System.getenv();

    if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
        if (cliParser.hasOption("app_attempt_id")) {
            String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
            appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
        } else {
            throw new IllegalArgumentException("Application Attempt Id not set in the environment");
        }
    } else {
        ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
        appAttemptID = containerId.getApplicationAttemptId();
    }

    if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
        throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HOST.name())) {
        throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
        throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_PORT.name())) {
        throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
    }

    LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId()
            + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId="
            + appAttemptID.getAttemptId());

    int qmMemory = Integer.parseInt(cliParser.getOptionValue("qm_memory", "512"));
    int qmVCores = Integer.parseInt(cliParser.getOptionValue("qm_vcores", "2"));
    int trMemory = Integer.parseInt(cliParser.getOptionValue("tr_memory", "1024"));
    int trVCores = Integer.parseInt(cliParser.getOptionValue("tr_vcores", "4"));
    int workerMemory = Integer.parseInt(cliParser.getOptionValue("worker_memory", "2048"));
    int workerVCores = Integer.parseInt(cliParser.getOptionValue("worker_vcores", "4"));

    int requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));

    String appMasterHostName = InetAddress.getLocalHost().getHostName();

    this.appContext = new AppContext(conf, appAttemptID, workerMemory, workerVCores, requestPriority,
            appMasterHostName);

    return true;
}

---

From source file:org.apache.tez.dag.app.rm.ContainerFactory.java

License:Apache License

public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) {
    this.nextId = new AtomicLong(1);
    ApplicationId appId = ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId());
    this.customAppAttemptId = ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId());
}

---

From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java

License:Apache License

/**
 * Mock {@link ApplicationAttemptId}//  w w  w . j ava2 s  . co  m
 *
 * @param appId the app id
 * @param attemptId the app attempt id
 * @return mocked {@link ApplicationAttemptId}
 */
public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) {
    ApplicationId applicationId = mock(ApplicationId.class);
    when(applicationId.getClusterTimestamp()).thenReturn(0L);
    when(applicationId.getId()).thenReturn(appId);
    ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class);
    when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
    when(applicationAttemptId.getAttemptId()).thenReturn(attemptId);
    return applicationAttemptId;
}

---

From source file:org.springframework.yarn.boot.MockUtils.java

License:Apache License

public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) {
    ApplicationId applicationId = mock(ApplicationId.class);
    when(applicationId.getClusterTimestamp()).thenReturn(0L);
    when(applicationId.getId()).thenReturn(appId);
    ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class);
    when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
    when(applicationAttemptId.getAttemptId()).thenReturn(attemptId);
    return applicationAttemptId;
}

---