Example usage for com.amazonaws.services.elasticmapreduce.model StepConfig getName

List of usage examples for com.amazonaws.services.elasticmapreduce.model StepConfig getName

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.elasticmapreduce.model.*
  5. StepConfig
  6. getName

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce.model StepConfig getName.

Prototype


public String getName()

Source Link

Document

The name of the step.

Usage

From source file:datameer.awstasks.aws.emr.EmrCluster.java

License:Apache License

public StepFuture executeJobStep(String name, File jobJar, String s3JobJarName, Class<?> mainClass,
        String... args) {/*from  w ww .jav  a 2s  .  c o  m*/
    checkConnection(true);
    HadoopJarStepConfig jarConfig = new HadoopJarStepConfig();
    if (jobJar != null) {
        String s3JobJarUri = uploadingJobJar(jobJar, s3JobJarName);
        jarConfig.setJar(s3JobJarUri);
    }
    if (mainClass != null) {
        jarConfig.setMainClass(mainClass.getName());
    }
    jarConfig.setArgs(Arrays.asList(args));
    StepConfig stepConfig = new StepConfig();
    stepConfig.setName(name);
    stepConfig.setActionOnFailure("CONTINUE");
    stepConfig.setHadoopJarStep(jarConfig);
    _emrWebService
            .addJobFlowSteps(new AddJobFlowStepsRequest().withJobFlowId(_jobFlowId).withSteps(stepConfig));
    _emrWebService.clearDescribeJobFlowCache();
    return new StepFuture(stepConfig.getName(), getStepIndex(getJobFlowDetail(_jobFlowId), name));
}

From source file:org.finra.dm.dao.impl.MockEmrOperationsImpl.java

License:Apache License

/**
 * Add Job Flow Step to AmazonElasticMapReduceClient
 *//*from   w  w  w . ja  v  a 2  s.c o m*/
@Override
public List<String> addJobFlowStepsRequest(AmazonElasticMapReduceClient emrClient,
        AddJobFlowStepsRequest addJobFlowStepsRequest) {
    if (addJobFlowStepsRequest.getSteps() != null && addJobFlowStepsRequest.getSteps().get(0) != null) {
        StepConfig firstStep = addJobFlowStepsRequest.getSteps().get(0);

        if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_BAD_REQUEST)) {
            AmazonServiceException badRequestException = new AmazonServiceException(
                    MockAwsOperationsHelper.AMAZON_BAD_REQUEST);
            badRequestException.setStatusCode(HttpStatus.SC_BAD_REQUEST);
            throw badRequestException;
        } else if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_NOT_FOUND)) {
            AmazonServiceException notFoundException = new AmazonServiceException(
                    MockAwsOperationsHelper.AMAZON_NOT_FOUND);
            notFoundException.setStatusCode(HttpStatus.SC_NOT_FOUND);
            throw notFoundException;
        } else if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION)) {
            throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        }
    }

    MockEmrJobFlow cluster = getClusterById(addJobFlowStepsRequest.getJobFlowId());
    if (cluster == null) {
        throw new AmazonServiceException(
                "No Cluster exists with jobFlowId: " + addJobFlowStepsRequest.getJobFlowId());
    }
    List<String> jobIds = new ArrayList<>();
    for (StepConfig step : addJobFlowStepsRequest.getSteps()) {
        jobIds.add(addClusterStep(cluster.getJobFlowId(), step).getJobFlowId());
    }
    return jobIds;
}

From source file:org.finra.dm.dao.impl.MockEmrOperationsImpl.java

License:Apache License

private MockEmrJobFlow addClusterStep(String jobFlowId, StepConfig step) {
    List<MockEmrJobFlow> mockSteps = getStepsByClusterId(jobFlowId);
    if (mockSteps == null) {
        mockSteps = new ArrayList<>();
    }/*from w w  w .j a  va 2s.c  o  m*/

    MockEmrJobFlow mockStep = new MockEmrJobFlow();
    mockStep.setJobFlowId(getNewJobFlowId());
    mockStep.setJobFlowName(step.getName());
    if (step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_NAME)) {
        mockStep.setStatus(StepState.RUNNING.toString());
    } else {
        mockStep.setStatus(StepState.PENDING.toString());
    }
    mockStep.setJarLocation(step.getHadoopJarStep().getJar());

    mockSteps.add(mockStep);
    setStepsByClusterId(jobFlowId, mockSteps);
    return mockStep;
}

From source file:org.finra.herd.dao.impl.MockEmrOperationsImpl.java

License:Apache License

private MockEmrJobFlow addClusterStep(String jobFlowId, StepConfig step) {
    List<MockEmrJobFlow> mockSteps = getStepsByClusterId(jobFlowId);
    if (mockSteps == null) {
        mockSteps = new ArrayList<>();
    }//from w  ww.  jav a 2 s . c  o m

    MockEmrJobFlow mockStep = new MockEmrJobFlow();
    if (!step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) {
        mockStep.setJobFlowId(getNewJobFlowId());
    }
    mockStep.setJobFlowName(step.getName());
    if (step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_NAME)
            || step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) {
        mockStep.setStatus(StepState.RUNNING.toString());
    } else {
        mockStep.setStatus(StepState.PENDING.toString());
    }
    mockStep.setJarLocation(step.getHadoopJarStep().getJar());

    mockSteps.add(mockStep);
    setStepsByClusterId(jobFlowId, mockSteps);
    return mockStep;
}

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONArray runningsJobFlow(AmazonElasticMapReduce emr) {
    List<JSONObject> l = new ArrayList<JSONObject>();

    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
            .withJobFlowStates(JOB_FLOW_STATUS_STARTING, JOB_FLOW_STATUS_RUNNING, JOB_FLOW_STATUS_WAITING);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) {
        JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
        Map<String, Object> m = new HashMap<String, Object>();
        m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
        m.put(Response.STATE, executionStatusDetail.getState());
        m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
        m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
        m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));

        if (!isEmpty(jobFlowDetail.getSteps())) {
            List<Object> ll = new ArrayList<Object>();
            for (StepDetail sd : jobFlowDetail.getSteps()) {
                Map<String, Object> mm = new HashMap<String, Object>();
                StepConfig sc = sd.getStepConfig();
                StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

                mm.put(Response.NAME, sc.getName());
                mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
                mm.put(Response.STATE, object2String(sesd.getState(), false));
                mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
                mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
                mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

                HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
                mm.put(Response.JAR, object2String(hjsc.getJar(), false));
                mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

                if (!isEmpty(hjsc.getArgs())) {
                    mm.put(Response.ARGS, hjsc.getArgs());
                }//from w w w  .  ja  v  a  2 s .  c om

                ll.add(mm);
            }
            m.put(Response.STEPS, ll);
        }

        l.add(new JSONObject(m));
    }

    return new JSONArray(l);
}

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) {
    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    if (describeJobFlowsResult.getJobFlows().size() != 1) {
        return new JSONObject();
    }/* w w w.j a va  2 s .  co  m*/

    JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
    JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
    JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();

    Map<String, Object> m = new HashMap<String, Object>();
    m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
    m.put(Response.STATE, executionStatusDetail.getState());
    m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
    m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
    m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));
    m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false));
    m.put(Response.NAME, jobFlowDetail.getName());
    m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false));

    if (!isEmpty(jobFlowDetail.getSupportedProducts())) {
        m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts());
    }

    m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false));
    m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false));
    m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false));
    m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount()));
    m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS,
            object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true));
    m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false));
    m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false));
    m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false));
    m.put(Response.AVAILABILITY_ZONE,
            object2String(instancesDetail.getPlacement().getAvailabilityZone(), false));
    m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false));

    if (!isEmpty(jobFlowDetail.getBootstrapActions())) {
        List<Object> l = new ArrayList<Object>();
        for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) {
            Map<String, Object> mm = new HashMap<String, Object>();

            BootstrapActionConfig bac = bad.getBootstrapActionConfig();
            ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction();

            mm.put(Response.NAME, object2String(bac.getName(), false));
            mm.put(Response.PATH, object2String(sbac.getPath(), false));

            if (!isEmpty(sbac.getArgs())) {
                mm.put(Response.ARGS, sbac.getArgs());
            }
            l.add(mm);
        }
        m.put(Response.BOOTSTRAP_ACTIONS, l);
    }

    if (!isEmpty(jobFlowDetail.getSteps())) {
        List<Object> l = new ArrayList<Object>();
        for (StepDetail sd : jobFlowDetail.getSteps()) {
            Map<String, Object> mm = new HashMap<String, Object>();
            StepConfig sc = sd.getStepConfig();
            StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

            mm.put(Response.NAME, sc.getName());
            mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
            mm.put(Response.STATE, object2String(sesd.getState(), false));
            mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
            mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
            mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

            HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
            mm.put(Response.JAR, object2String(hjsc.getJar(), false));
            mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

            if (!isEmpty(hjsc.getArgs())) {
                mm.put(Response.ARGS, hjsc.getArgs());
            }

            l.add(mm);
        }
        m.put(Response.STEPS, l);
    }

    return new JSONObject(m);
}