Example usage for com.amazonaws.services.elasticmapreduce.model JobFlowInstancesDetail getMasterPublicDnsName

List of usage examples for com.amazonaws.services.elasticmapreduce.model JobFlowInstancesDetail getMasterPublicDnsName

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.elasticmapreduce.model.*
  5. JobFlowInstancesDetail
  6. getMasterPublicDnsName

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce.model JobFlowInstancesDetail getMasterPublicDnsName.

Prototype


public String getMasterPublicDnsName()

Source Link

Document

The DNS name of the master node.

Usage

From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java

License:Apache License

/**
 * @param config//from  w  w  w .ja  v  a2s .c o  m
 * @throws URISyntaxException
 */
public void runJob(Config config) throws URISyntaxException {
    RunJobFlowRequest runJobFlowRequest = null;

    CreateStepConfigger csc = getCreateStepConfigger(config);
    if (csc == null) {
        log.error("Step config create error");
        return;
    }

    if (jobFlowId == null) {
        runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME)
                .withBootstrapActions(
                        new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)),
                        new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI)
                                        .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")),
                        new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path())))
                .withInstances(setupJobFlowInstancesConfig());
        if (!isEmpty(emrProperties.getLogUri())) {
            runJobFlowRequest.setLogUri(emrProperties.getLogUri());
        }

        List<StepConfig> stepConfigs = new ArrayList<StepConfig>();
        if (emrProperties.isDebug()) {
            StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME)
                    .withActionOnFailure(ACTION_ON_TERMINATE)
                    .withHadoopJarStep(new StepFactory().newEnableDebuggingStep());
            stepConfigs.add(enableDebugging);
        }

        for (StepConfig sc : csc.createStepConfig(config)) {
            stepConfigs.add(sc);
        }
        runJobFlowRequest.setSteps(stepConfigs);

        try {
            RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest);
            jobFlowId = result.getJobFlowId();
            checkDate = new Date();
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    } else {
        AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId)
                .withSteps(csc.createStepConfig(config));
        emr.addJobFlowSteps(addJobFlowStepsRequest);
    }

    running = true;
    try {
        config.setJobFlowId(jobFlowId);
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }

    int stepSize = 0;
    String stepStatus = JobUtils.STEP_STATUS_PENDING;
    while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) {
        if (sleep()) {
            break;
        }

        DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
                .withJobFlowIds(jobFlowId);
        DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
        if (describeJobFlowsResult.getJobFlows().size() != 1) {
            break;
        }

        JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
        JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();
        masterPublicDnsName = instancesDetail.getMasterPublicDnsName();
        if (isEmpty(config.getMasterPublicDnsName())) {
            try {
                config.setMasterPublicDnsName(masterPublicDnsName);
                QueueUtils.updateQueue(config);
            } catch (IOException e) {
                e.printStackTrace();
                log.error(e);
            }
        }

        stepSize = jobFlowDetail.getSteps().size();
        for (StepDetail stepDetail : jobFlowDetail.getSteps()) {
            if (stepDetail.getStepConfig().getName().equals(config.getName())) {
                stepStatus = stepDetail.getExecutionStatusDetail().getState();
                break;
            }
        }
    }

    if (config.isDeleteOnExit()) {
        if (config.getJobType() == Config.JOB_TYPE_STREAMING) {
            S3Utils.delete(s3, config.getArgMap().get("mapper"));
            S3Utils.delete(s3, config.getArgMap().get("reducer"));
        } else {
            S3Utils.delete(s3, config.getRun());
        }
    }

    // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV)
    if (stepSize >= 255) {
        instanceTerminate();
    }

    running = false;

    if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) {
        config.setStatus(Config.JOB_STATUS_COMPLETE);
    } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) {
        config.setStatus(Config.JOB_STATUS_ERROR);
    } else if (terminated) {
        config.setStatus(Config.JOB_STATUS_CANCEL);
    }

    try {
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }
}

From source file:org.huahinframework.emanager.util.JobUtils.java

License:Apache License

public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) {
    DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow);
    DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
    if (describeJobFlowsResult.getJobFlows().size() != 1) {
        return new JSONObject();
    }//  www .j  av a 2s.  co m

    JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
    JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail();
    JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();

    Map<String, Object> m = new HashMap<String, Object>();
    m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId());
    m.put(Response.STATE, executionStatusDetail.getState());
    m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString());
    m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true));
    m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true));
    m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false));
    m.put(Response.NAME, jobFlowDetail.getName());
    m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false));

    if (!isEmpty(jobFlowDetail.getSupportedProducts())) {
        m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts());
    }

    m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false));
    m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false));
    m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false));
    m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount()));
    m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS,
            object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true));
    m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false));
    m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false));
    m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false));
    m.put(Response.AVAILABILITY_ZONE,
            object2String(instancesDetail.getPlacement().getAvailabilityZone(), false));
    m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false));

    if (!isEmpty(jobFlowDetail.getBootstrapActions())) {
        List<Object> l = new ArrayList<Object>();
        for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) {
            Map<String, Object> mm = new HashMap<String, Object>();

            BootstrapActionConfig bac = bad.getBootstrapActionConfig();
            ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction();

            mm.put(Response.NAME, object2String(bac.getName(), false));
            mm.put(Response.PATH, object2String(sbac.getPath(), false));

            if (!isEmpty(sbac.getArgs())) {
                mm.put(Response.ARGS, sbac.getArgs());
            }
            l.add(mm);
        }
        m.put(Response.BOOTSTRAP_ACTIONS, l);
    }

    if (!isEmpty(jobFlowDetail.getSteps())) {
        List<Object> l = new ArrayList<Object>();
        for (StepDetail sd : jobFlowDetail.getSteps()) {
            Map<String, Object> mm = new HashMap<String, Object>();
            StepConfig sc = sd.getStepConfig();
            StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail();

            mm.put(Response.NAME, sc.getName());
            mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure());
            mm.put(Response.STATE, object2String(sesd.getState(), false));
            mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true));
            mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true));
            mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true));

            HadoopJarStepConfig hjsc = sc.getHadoopJarStep();
            mm.put(Response.JAR, object2String(hjsc.getJar(), false));
            mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false));

            if (!isEmpty(hjsc.getArgs())) {
                mm.put(Response.ARGS, hjsc.getArgs());
            }

            l.add(mm);
        }
        m.put(Response.STEPS, l);
    }

    return new JSONObject(m);
}