Example usage for com.amazonaws.services.elasticmapreduce.model DescribeJobFlowsResult getJobFlows

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce.model DescribeJobFlowsResult getJobFlows.

Prototype

public java.util.List<JobFlowDetail> getJobFlows() 

Source Link

Document

A list of job flows matching the parameters supplied.

Usage

From source file:awswc.AwsConsoleApp.java

License:Open Source License

public static void wasFinished(RunJobFlowResult runJobFlowResult) throws InterruptedException {
    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
            .withJobFlowIds(runJobFlowResult.getJobFlowId());
    int secondsBetweenPolling = 120;
    String state = null;//from ww w  .  ja v a  2  s  . c o m
    do {
        Thread.sleep(secondsBetweenPolling * 1000);
        DescribeJobFlowsResult jobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
        JobFlowDetail detail = jobFlowsResult.getJobFlows().get(0);
        JobFlowExecutionStatusDetail executionStatusDetail = detail.getExecutionStatusDetail();
        state = executionStatusDetail.getState();
    } while (state != null && !state.equals("COMPLETE") && !state.equals("FAILED")
            && !state.equals("TERMINATED"));
}

---

From source file:com.clouddrive.parth.NewClass.java

public static String runCluster() throws Exception {
    long start = System.currentTimeMillis();
    String temp = "";
    // Configure the job flow
    //RunJobFlowRequest request = new RunJobFlowRequest().withName("parth");
    // if (request == null) {
    RunJobFlowRequest request = new RunJobFlowRequest(FLOW_NAME, configInstance());
    request.setLogUri(S3N_LOG_URI);
    // }/*  ww  w  .  ja  v  a 2 s .com*/

    // Configure the Hadoop jar to use
    HadoopJarStepConfig jarConfig = new HadoopJarStepConfig(S3N_HADOOP_JAR);
    jarConfig.setArgs(ARGS_AS_LIST);

    try {

        StepConfig enableDebugging = new StepConfig().withName("Enable debugging")
                .withActionOnFailure("TERMINATE_JOB_FLOW")
                .withHadoopJarStep(new StepFactory().newEnableDebuggingStep());

        StepConfig runJar = new StepConfig(S3N_HADOOP_JAR.substring(S3N_HADOOP_JAR.indexOf('/') + 1),
                jarConfig);

        request.setSteps(Arrays.asList(new StepConfig[] { enableDebugging, runJar }));

        // Run the job flow
        RunJobFlowResult result = emr.runJobFlow(request);

        // Check the status of the running job
        String lastState = "";

        STATUS_LOOP: while (true) {
            DescribeJobFlowsRequest desc = new DescribeJobFlowsRequest(
                    Arrays.asList(new String[] { result.getJobFlowId() }));
            DescribeJobFlowsResult descResult = emr.describeJobFlows(desc);
            for (JobFlowDetail detail : descResult.getJobFlows()) {
                String state = detail.getExecutionStatusDetail().getState();
                if (isDone(state)) {
                    System.out.println("Job " + state + ": " + detail.toString());
                    break STATUS_LOOP;
                } else if (!lastState.equals(state)) {
                    lastState = state;
                    System.out.println("Job " + state + " at " + new Date().toString());
                }
            }
            Thread.sleep(10000);
        }
        temp = FLOW_NAME;
        long end = System.currentTimeMillis();
        System.out.println("Computation " + (end - start));
    } catch (AmazonServiceException ase) {
        System.out.println("Caught Exception: " + ase.getMessage());
        System.out.println("Reponse Status Code: " + ase.getStatusCode());
        System.out.println("Error Code: " + ase.getErrorCode());
        System.out.println("Request ID: " + ase.getRequestId());
    }
    return temp;
}

---

From source file:com.tstordyallison.ffmpegmr.emr.JobflowConfiguration.java

License:Open Source License

public JobflowConfiguration() {
    try {/*w w w  . j a  va2 s.c om*/
        Gson gson = new GsonBuilder().disableHtmlEscaping().create();
        jobflow = gson.fromJson(new BufferedReader(new FileReader(JOB_FLOW_INFO_FILE)),
                JobflowConfiguration.JobFlow.class);
        instance = gson.fromJson(new BufferedReader(new FileReader(INSTANCE_INFO_FILE)),
                JobflowConfiguration.Instance.class);

        // We must also go and get an up to date value for the coreInstanceCount using the API.
        DescribeJobFlowsResult jobFlowDescription = JobController.getEmr()
                .describeJobFlows(new DescribeJobFlowsRequest().withJobFlowIds(jobflow.jobFlowId));
        JobFlowDetail jobFlowDetail = jobFlowDescription.getJobFlows().get(0);
        Collection<InstanceGroupDetail> instanceGroups = jobFlowDetail.getInstances().getInstanceGroups();

        for (InstanceGroupDetail instanceGroup : instanceGroups) {
            if (instanceGroup.getInstanceRole().equals("CORE")) {
                jobflow.coreInstanceCount = instanceGroup.getInstanceRunningCount();
                break;
            }
        }

    } catch (Exception e) {
        System.err.println("WARNING: Unable to read instance configuration files - logs will be inaccurate.");
    }
}

---

From source file:datameer.awstasks.aws.emr.EmrCluster.java

License:Apache License

protected JobFlowDetail getJobFlowDetail(String jobFlowId) {
    DescribeJobFlowsResult describeJobFlows = _emrWebService
            .describeJobFlows(new DescribeJobFlowsRequest().withJobFlowIds(jobFlowId));
    List<JobFlowDetail> jobFlows = describeJobFlows.getJobFlows();
    if (jobFlows.isEmpty()) {
        throw new IllegalArgumentException("no job flow with id '" + _jobFlowId + "' found");
    }//from  ww w .  j a va 2 s.c  om
    return jobFlows.get(0);
}

---

From source file:datameer.awstasks.aws.emr.EmrCluster.java

License:Apache License

protected List<JobFlowDetail> getRunningJobFlowDetailsByName(String name) {
    DescribeJobFlowsResult describeJobFlows = _emrWebService.describeJobFlows(new DescribeJobFlowsRequest()
            .withJobFlowStates(JobFlowState.STARTING.name(), JobFlowState.BOOTSTRAPPING.name(),
                    JobFlowState.WAITING.name(), JobFlowState.RUNNING.name()));
    List<JobFlowDetail> jobFlows = describeJobFlows.getJobFlows();
    for (Iterator<JobFlowDetail> iterator = jobFlows.iterator(); iterator.hasNext();) {
        JobFlowDetail jobFlowDetail = iterator.next();
        if (!name.equals(jobFlowDetail.getName())) {
            iterator.remove();/*from   w ww .ja va2s . co m*/
        }
    }
    return jobFlows;
}

---

From source file:fr.ens.biologie.genomique.eoulsan.util.cloud.AWSElasticMapReduceJob.java

License:LGPL

/**
 * Wait the end of the job/*from   w w w .j  a  v a  2  s . com*/
 * @param secondBetweenChecking number of seconds to wait between 2 checks
 * @return the final state of the job
 */
public String waitForJob(final int secondBetweenChecking) {

    if (this.runFlowResult == null) {
        return null;
    }

    final DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
            .withJobFlowIds(this.runFlowResult.getJobFlowId());

    String state = null;
    String lastState = null;
    int failCount = 0;

    try {

        do {

            Thread.sleep(secondBetweenChecking * 1000);

            try {
                final DescribeJobFlowsResult jobFlowsResult = this.elasticMapReduceClient
                        .describeJobFlows(describeJobFlowsRequest);
                final JobFlowDetail detail = jobFlowsResult.getJobFlows().get(0);
                final JobFlowExecutionStatusDetail executionStatusDetail = detail.getExecutionStatusDetail();
                failCount = 0;

                state = executionStatusDetail.getState();
            } catch (AmazonClientException ace) {

                failCount++;
                getLogger().warning("Amazon client exception: " + ace.getMessage());

                if (failCount >= MAX_FAIL_COUNT) {
                    throw ace;
                }

            }

            if (lastState == null || !lastState.equals(state)) {

                getLogger().info("State of the job " + this.runFlowResult.getJobFlowId() + ": " + state);
                lastState = state;
            }

        } while (state != null && !state.equals("COMPLETED") && !state.equals("FAILED")
                && !state.equals("TERMINATED"));

        return state;

    } catch (InterruptedException e) {
        getLogger().warning("Error while waiting AWS Elastic MapReduce Job: " + e.getMessage());
    }

    return null;
}

---

From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java

License:Apache License

/**
 * @param config//from  w ww .jav  a2 s .c om
 * @throws URISyntaxException
 */
public void runJob(Config config) throws URISyntaxException {
    RunJobFlowRequest runJobFlowRequest = null;

    CreateStepConfigger csc = getCreateStepConfigger(config);
    if (csc == null) {
        log.error("Step config create error");
        return;
    }

    if (jobFlowId == null) {
        runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME)
                .withBootstrapActions(
                        new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)),
                        new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI)
                                        .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")),
                        new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path())))
                .withInstances(setupJobFlowInstancesConfig());
        if (!isEmpty(emrProperties.getLogUri())) {
            runJobFlowRequest.setLogUri(emrProperties.getLogUri());
        }

        List<StepConfig> stepConfigs = new ArrayList<StepConfig>();
        if (emrProperties.isDebug()) {
            StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME)
                    .withActionOnFailure(ACTION_ON_TERMINATE)
                    .withHadoopJarStep(new StepFactory().newEnableDebuggingStep());
            stepConfigs.add(enableDebugging);
        }

        for (StepConfig sc : csc.createStepConfig(config)) {
            stepConfigs.add(sc);
        }
        runJobFlowRequest.setSteps(stepConfigs);

        try {
            RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest);
            jobFlowId = result.getJobFlowId();
            checkDate = new Date();
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    } else {
        AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId)
                .withSteps(csc.createStepConfig(config));
        emr.addJobFlowSteps(addJobFlowStepsRequest);
    }

    running = true;
    try {
        config.setJobFlowId(jobFlowId);
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }

    int stepSize = 0;
    String stepStatus = JobUtils.STEP_STATUS_PENDING;
    while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) {
        if (sleep()) {
            break;
        }

        DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
                .withJobFlowIds(jobFlowId);
        DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
        if (describeJobFlowsResult.getJobFlows().size() != 1) {
            break;
        }

        JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
        JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();
        masterPublicDnsName = instancesDetail.getMasterPublicDnsName();
        if (isEmpty(config.getMasterPublicDnsName())) {
            try {
                config.setMasterPublicDnsName(masterPublicDnsName);
                QueueUtils.updateQueue(config);
            } catch (IOException e) {
                e.printStackTrace();
                log.error(e);
            }
        }

        stepSize = jobFlowDetail.getSteps().size();
        for (StepDetail stepDetail : jobFlowDetail.getSteps()) {
            if (stepDetail.getStepConfig().getName().equals(config.getName())) {
                stepStatus = stepDetail.getExecutionStatusDetail().getState();
                break;
            }
        }
    }

    if (config.isDeleteOnExit()) {
        if (config.getJobType() == Config.JOB_TYPE_STREAMING) {
            S3Utils.delete(s3, config.getArgMap().get("mapper"));
            S3Utils.delete(s3, config.getArgMap().get("reducer"));
        } else {
            S3Utils.delete(s3, config.getRun());
        }
    }

    // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV)
    if (stepSize >= 255) {
        instanceTerminate();
    }

    running = false;

    if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) {
        config.setStatus(Config.JOB_STATUS_COMPLETE);
    } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) {
        config.setStatus(Config.JOB_STATUS_ERROR);
    } else if (terminated) {
        config.setStatus(Config.JOB_STATUS_CANCEL);
    }

    try {
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }
}

---

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONArray listJobFlow(AmazonElasticMapReduce emr) {
    List<JSONObject> l = new ArrayList<JSONObject>();

    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(new DescribeJobFlowsRequest());
    for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) {
        JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
        Map<String, String> m = new HashMap<String, String>();
        m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
        m.put(Response.STATE, executionStatusDetail.getState());
        m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
        m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
        m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));
        l.add(new JSONObject(m));
    }/*from  w  w w .ja va2 s . c o m*/

    return new JSONArray(l);
}

---

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONArray runningsJobFlow(AmazonElasticMapReduce emr) {
    List<JSONObject> l = new ArrayList<JSONObject>();

    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
            .withJobFlowStates(JOB_FLOW_STATUS_STARTING, JOB_FLOW_STATUS_RUNNING, JOB_FLOW_STATUS_WAITING);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) {
        JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
        Map<String, Object> m = new HashMap<String, Object>();
        m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
        m.put(Response.STATE, executionStatusDetail.getState());
        m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
        m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
        m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));

        if (!isEmpty(jobFlowDetail.getSteps())) {
            List<Object> ll = new ArrayList<Object>();
            for (StepDetail sd : jobFlowDetail.getSteps()) {
                Map<String, Object> mm = new HashMap<String, Object>();
                StepConfig sc = sd.getStepConfig();
                StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

                mm.put(Response.NAME, sc.getName());
                mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
                mm.put(Response.STATE, object2String(sesd.getState(), false));
                mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
                mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
                mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

                HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
                mm.put(Response.JAR, object2String(hjsc.getJar(), false));
                mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

                if (!isEmpty(hjsc.getArgs())) {
                    mm.put(Response.ARGS, hjsc.getArgs());
                }/*from  w w  w.  j a va 2s.c  om*/

                ll.add(mm);
            }
            m.put(Response.STEPS, ll);
        }

        l.add(new JSONObject(m));
    }

    return new JSONArray(l);
}

---

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) {
    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    if (describeJobFlowsResult.getJobFlows().size() != 1) {
        return new JSONObject();
    }/*from   w w  w. j  ava  2  s  .c  o m*/

    JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
    JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
    JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();

    Map<String, Object> m = new HashMap<String, Object>();
    m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
    m.put(Response.STATE, executionStatusDetail.getState());
    m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
    m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
    m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));
    m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false));
    m.put(Response.NAME, jobFlowDetail.getName());
    m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false));

    if (!isEmpty(jobFlowDetail.getSupportedProducts())) {
        m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts());
    }

    m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false));
    m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false));
    m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false));
    m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount()));
    m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS,
            object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true));
    m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false));
    m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false));
    m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false));
    m.put(Response.AVAILABILITY_ZONE,
            object2String(instancesDetail.getPlacement().getAvailabilityZone(), false));
    m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false));

    if (!isEmpty(jobFlowDetail.getBootstrapActions())) {
        List<Object> l = new ArrayList<Object>();
        for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) {
            Map<String, Object> mm = new HashMap<String, Object>();

            BootstrapActionConfig bac = bad.getBootstrapActionConfig();
            ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction();

            mm.put(Response.NAME, object2String(bac.getName(), false));
            mm.put(Response.PATH, object2String(sbac.getPath(), false));

            if (!isEmpty(sbac.getArgs())) {
                mm.put(Response.ARGS, sbac.getArgs());
            }
            l.add(mm);
        }
        m.put(Response.BOOTSTRAP_ACTIONS, l);
    }

    if (!isEmpty(jobFlowDetail.getSteps())) {
        List<Object> l = new ArrayList<Object>();
        for (StepDetail sd : jobFlowDetail.getSteps()) {
            Map<String, Object> mm = new HashMap<String, Object>();
            StepConfig sc = sd.getStepConfig();
            StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

            mm.put(Response.NAME, sc.getName());
            mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
            mm.put(Response.STATE, object2String(sesd.getState(), false));
            mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
            mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
            mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

            HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
            mm.put(Response.JAR, object2String(hjsc.getJar(), false));
            mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

            if (!isEmpty(hjsc.getArgs())) {
                mm.put(Response.ARGS, hjsc.getArgs());
            }

            l.add(mm);
        }
        m.put(Response.STEPS, l);
    }

    return new JSONObject(m);
}

---

• «
• 1
• 2
• »