Example usage for com.amazonaws.services.elasticmapreduce.model StepDetail getExecutionStatusDetail

List of usage examples for com.amazonaws.services.elasticmapreduce.model StepDetail getExecutionStatusDetail

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.elasticmapreduce.model.*
  5. StepDetail
  6. getExecutionStatusDetail

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce.model StepDetail getExecutionStatusDetail.

Prototype


public StepExecutionStatusDetail getExecutionStatusDetail()

Source Link

Document

The description of the step status.

Usage

From source file:datameer.awstasks.aws.emr.EmrCluster.java

License:Apache License

protected StepState getStepState(final String jobFlowId, final String stepName) {
    JobFlowDetail flowDetail = getJobFlowDetail(jobFlowId);
    StepDetail stepDetail = getStepDetail(flowDetail, stepName);
    StepState stepState = StepState.valueOf(stepDetail.getExecutionStatusDetail().getState());
    return stepState;
}

From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java

License:Apache License

/**
 * @param config//from  ww  w  .ja  v a 2 s  .c  om
 * @throws URISyntaxException
 */
public void runJob(Config config) throws URISyntaxException {
    RunJobFlowRequest runJobFlowRequest = null;

    CreateStepConfigger csc = getCreateStepConfigger(config);
    if (csc == null) {
        log.error("Step config create error");
        return;
    }

    if (jobFlowId == null) {
        runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME)
                .withBootstrapActions(
                        new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)),
                        new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI)
                                        .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")),
                        new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path())))
                .withInstances(setupJobFlowInstancesConfig());
        if (!isEmpty(emrProperties.getLogUri())) {
            runJobFlowRequest.setLogUri(emrProperties.getLogUri());
        }

        List<StepConfig> stepConfigs = new ArrayList<StepConfig>();
        if (emrProperties.isDebug()) {
            StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME)
                    .withActionOnFailure(ACTION_ON_TERMINATE)
                    .withHadoopJarStep(new StepFactory().newEnableDebuggingStep());
            stepConfigs.add(enableDebugging);
        }

        for (StepConfig sc : csc.createStepConfig(config)) {
            stepConfigs.add(sc);
        }
        runJobFlowRequest.setSteps(stepConfigs);

        try {
            RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest);
            jobFlowId = result.getJobFlowId();
            checkDate = new Date();
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    } else {
        AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId)
                .withSteps(csc.createStepConfig(config));
        emr.addJobFlowSteps(addJobFlowStepsRequest);
    }

    running = true;
    try {
        config.setJobFlowId(jobFlowId);
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }

    int stepSize = 0;
    String stepStatus = JobUtils.STEP_STATUS_PENDING;
    while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) {
        if (sleep()) {
            break;
        }

        DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
                .withJobFlowIds(jobFlowId);
        DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
        if (describeJobFlowsResult.getJobFlows().size() != 1) {
            break;
        }

        JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
        JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();
        masterPublicDnsName = instancesDetail.getMasterPublicDnsName();
        if (isEmpty(config.getMasterPublicDnsName())) {
            try {
                config.setMasterPublicDnsName(masterPublicDnsName);
                QueueUtils.updateQueue(config);
            } catch (IOException e) {
                e.printStackTrace();
                log.error(e);
            }
        }

        stepSize = jobFlowDetail.getSteps().size();
        for (StepDetail stepDetail : jobFlowDetail.getSteps()) {
            if (stepDetail.getStepConfig().getName().equals(config.getName())) {
                stepStatus = stepDetail.getExecutionStatusDetail().getState();
                break;
            }
        }
    }

    if (config.isDeleteOnExit()) {
        if (config.getJobType() == Config.JOB_TYPE_STREAMING) {
            S3Utils.delete(s3, config.getArgMap().get("mapper"));
            S3Utils.delete(s3, config.getArgMap().get("reducer"));
        } else {
            S3Utils.delete(s3, config.getRun());
        }
    }

    // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV)
    if (stepSize >= 255) {
        instanceTerminate();
    }

    running = false;

    if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) {
        config.setStatus(Config.JOB_STATUS_COMPLETE);
    } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) {
        config.setStatus(Config.JOB_STATUS_ERROR);
    } else if (terminated) {
        config.setStatus(Config.JOB_STATUS_CANCEL);
    }

    try {
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }
}

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONArray runningsJobFlow(AmazonElasticMapReduce emr) {
    List<JSONObject> l = new ArrayList<JSONObject>();

    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
            .withJobFlowStates(JOB_FLOW_STATUS_STARTING, JOB_FLOW_STATUS_RUNNING, JOB_FLOW_STATUS_WAITING);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) {
        JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
        Map<String, Object> m = new HashMap<String, Object>();
        m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
        m.put(Response.STATE, executionStatusDetail.getState());
        m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
        m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
        m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));

        if (!isEmpty(jobFlowDetail.getSteps())) {
            List<Object> ll = new ArrayList<Object>();
            for (StepDetail sd : jobFlowDetail.getSteps()) {
                Map<String, Object> mm = new HashMap<String, Object>();
                StepConfig sc = sd.getStepConfig();
                StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

                mm.put(Response.NAME, sc.getName());
                mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
                mm.put(Response.STATE, object2String(sesd.getState(), false));
                mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
                mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
                mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

                HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
                mm.put(Response.JAR, object2String(hjsc.getJar(), false));
                mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

                if (!isEmpty(hjsc.getArgs())) {
                    mm.put(Response.ARGS, hjsc.getArgs());
                }/*ww w .  j a  v  a2 s.  c o m*/

                ll.add(mm);
            }
            m.put(Response.STEPS, ll);
        }

        l.add(new JSONObject(m));
    }

    return new JSONArray(l);
}

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) {
    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    if (describeJobFlowsResult.getJobFlows().size() != 1) {
        return new JSONObject();
    }//from   ww  w .  j a va 2s .c o  m

    JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
    JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
    JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();

    Map<String, Object> m = new HashMap<String, Object>();
    m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
    m.put(Response.STATE, executionStatusDetail.getState());
    m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
    m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
    m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));
    m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false));
    m.put(Response.NAME, jobFlowDetail.getName());
    m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false));

    if (!isEmpty(jobFlowDetail.getSupportedProducts())) {
        m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts());
    }

    m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false));
    m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false));
    m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false));
    m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount()));
    m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS,
            object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true));
    m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false));
    m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false));
    m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false));
    m.put(Response.AVAILABILITY_ZONE,
            object2String(instancesDetail.getPlacement().getAvailabilityZone(), false));
    m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false));

    if (!isEmpty(jobFlowDetail.getBootstrapActions())) {
        List<Object> l = new ArrayList<Object>();
        for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) {
            Map<String, Object> mm = new HashMap<String, Object>();

            BootstrapActionConfig bac = bad.getBootstrapActionConfig();
            ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction();

            mm.put(Response.NAME, object2String(bac.getName(), false));
            mm.put(Response.PATH, object2String(sbac.getPath(), false));

            if (!isEmpty(sbac.getArgs())) {
                mm.put(Response.ARGS, sbac.getArgs());
            }
            l.add(mm);
        }
        m.put(Response.BOOTSTRAP_ACTIONS, l);
    }

    if (!isEmpty(jobFlowDetail.getSteps())) {
        List<Object> l = new ArrayList<Object>();
        for (StepDetail sd : jobFlowDetail.getSteps()) {
            Map<String, Object> mm = new HashMap<String, Object>();
            StepConfig sc = sd.getStepConfig();
            StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

            mm.put(Response.NAME, sc.getName());
            mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
            mm.put(Response.STATE, object2String(sesd.getState(), false));
            mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
            mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
            mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

            HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
            mm.put(Response.JAR, object2String(hjsc.getJar(), false));
            mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

            if (!isEmpty(hjsc.getArgs())) {
                mm.put(Response.ARGS, hjsc.getArgs());
            }

            l.add(mm);
        }
        m.put(Response.STEPS, l);
    }

    return new JSONObject(m);
}