Example usage for com.amazonaws.services.elasticmapreduce.model ScriptBootstrapActionConfig ScriptBootstrapActionConfig

List of usage examples for com.amazonaws.services.elasticmapreduce.model ScriptBootstrapActionConfig ScriptBootstrapActionConfig

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.elasticmapreduce.model.*
  5. ScriptBootstrapActionConfig
  6. ScriptBootstrapActionConfig

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce.model ScriptBootstrapActionConfig ScriptBootstrapActionConfig.

Prototype

public ScriptBootstrapActionConfig()

Source Link

Document

Default constructor for ScriptBootstrapActionConfig object.

Usage

From source file:com.aegeus.aws.ElasticMapReduceService.java

License:Apache License

/**
 * Create a new EMR Cluster over Hadoop 2.4.0
 *//*from ww  w. j a v a 2s . c  o  m*/
public void createCluster() {
    JobFlowInstancesConfig instances = new JobFlowInstancesConfig()
            .withInstanceCount((int) config.getInstanceCount()).withMasterInstanceType(config.getMasterType())
            .withSlaveInstanceType(config.getSlaveType());

    if (Strings.isNullOrEmpty(config.getKeyName())) {
        instances.setEc2KeyName(config.getKeyName());
    }

    if (!Strings.isNullOrEmpty(config.getSubnetId())) {
        instances.setEc2SubnetId(config.getSubnetId());
    } else {
        instances.setPlacement(new PlacementType(config.getPlace()));
    }

    ScriptBootstrapActionConfig installEsConfig = new ScriptBootstrapActionConfig()
            .withPath("s3://support.elasticmapreduce/bootstrap-actions/other/elasticsearch_install.rb");

    BootstrapActionConfig installEs = new BootstrapActionConfig("Elasticsearch Install", installEsConfig);

    RunJobFlowRequest request = new RunJobFlowRequest().withName(config.getName()).withReleaseLabel("emr-4.1.0")
            .withServiceRole("Default_AWS_Role").withJobFlowRole("Default_AWS_Role")
            .withBootstrapActions(installEs).withInstances(instances);

    if (!Strings.isNullOrEmpty(config.getLogBucket())) {
        request.setLogUri(config.getLogBucket());
    }

    RunJobFlowResult result = emr.runJobFlow(request);

    clusterId = result.getJobFlowId();
}

From source file:datameer.awstasks.ant.emr.model.BootstrapConfig.java

License:Apache License

public BootstrapActionConfig createBootstrapActionConfig() {
    return new BootstrapActionConfig().withName(_name)
            .withScriptBootstrapAction(new ScriptBootstrapActionConfig().withPath(_path));
}

From source file:fr.ens.biologie.genomique.eoulsan.util.cloud.AWSElasticMapReduceJob.java

License:LGPL

void init() {

    requireNonNull(this.AWSAccessKey);
    requireNonNull(this.AWSAccessKey);
    requireNonNull(this.jarLocation);
    requireNonNull(this.jarArguments);
    requireNonNull(this.slavesInstanceType);
    requireNonNull(this.hadoopVersion);
    requireNonNull(this.jobFlowName);

    if (this.nInstances < 1) {
        throw new IllegalArgumentException("the number of instance is lower than 1");
    }//w ww  . j a v a 2s .  c  o  m

    if (this.masterInstanceType == null) {
        this.masterInstanceType = this.slavesInstanceType;
    }

    // Set the hadoop jar step
    final HadoopJarStepConfig hadoopJarStep = new HadoopJarStepConfig().withJar(this.jarLocation.trim())
            .withArgs(this.jarArguments);

    // Set step config
    final StepConfig stepConfig = new StepConfig().withName(this.jobFlowName + "-step")
            .withHadoopJarStep(hadoopJarStep).withActionOnFailure("TERMINATE_JOB_FLOW");

    // Set the instance
    final JobFlowInstancesConfig instances = new JobFlowInstancesConfig().withInstanceCount(this.nInstances)
            .withMasterInstanceType(this.masterInstanceType).withSlaveInstanceType(this.slavesInstanceType)
            .withHadoopVersion(this.hadoopVersion);

    // Configure hadoop
    final ScriptBootstrapActionConfig scriptBootstrapAction = new ScriptBootstrapActionConfig()
            .withPath("s3n://eu-west-1.elasticmapreduce/bootstrap-actions/configure-hadoop")
            .withArgs("--site-key-value",
                    "mapreduce.tasktracker.map.tasks.maximum=" + this.taskTrackerMaxMapTasks);

    final BootstrapActionConfig bootstrapActions = new BootstrapActionConfig().withName("Configure hadoop")
            .withScriptBootstrapAction(scriptBootstrapAction);

    // Enable debugging
    StepFactory stepFactory = new StepFactory();
    StepConfig enableDebugging = new StepConfig().withName("Enable Debugging")
            .withActionOnFailure("TERMINATE_JOB_FLOW").withHadoopJarStep(stepFactory.newEnableDebuggingStep());

    // Run flow
    this.runFlowRequest = new RunJobFlowRequest().withName(this.jobFlowName);

    // Enable or not debugging
    if (this.enableDebugging) {
        this.runFlowRequest.withInstances(instances).withSteps(enableDebugging, stepConfig);
    } else {
        this.runFlowRequest.withInstances(instances).withSteps(stepConfig);
    }

    // Limit the number of task in a task tracker
    if (this.taskTrackerMaxMapTasks > 0) {
        this.runFlowRequest.withBootstrapActions(bootstrapActions);
    }

    if (this.logPathname != null && !"".equals(this.logPathname)) {
        this.runFlowRequest.withLogUri(this.logPathname);
    }

    // Set EC2 Key name
    if (this.ec2KeyName != null) {
        this.runFlowRequest.getInstances().setEc2KeyName(this.ec2KeyName);
    }
}

From source file:org.finra.dm.dao.impl.EmrDaoImpl.java

License:Apache License

/**
 * Create the BootstrapActionConfig object from the bootstrap script.
 *
 * @param scriptDescription bootstrap script name to be displayed.
 * @param bootstrapScript location of the bootstrap script.
 *
 * @return bootstrap action configuration that contains all the bootstrap actions for the given configuration.
 *//*from   ww  w  .  j  av a  2s  . c  o  m*/
private BootstrapActionConfig getBootstrapActionConfig(String scriptDescription, String bootstrapScript) {
    // Create the BootstrapActionConfig object
    BootstrapActionConfig bootstrapConfig = new BootstrapActionConfig();
    ScriptBootstrapActionConfig bootstrapConfigScript = new ScriptBootstrapActionConfig();

    // Set the bootstrapScript
    bootstrapConfig.setName(scriptDescription);
    bootstrapConfigScript.setPath(bootstrapScript);
    bootstrapConfig.setScriptBootstrapAction(bootstrapConfigScript);

    // Return the object
    return bootstrapConfig;
}

From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java

License:Apache License

/**
 * @param config/*  w w w.  ja  va 2s. c  om*/
 * @throws URISyntaxException
 */
public void runJob(Config config) throws URISyntaxException {
    RunJobFlowRequest runJobFlowRequest = null;

    CreateStepConfigger csc = getCreateStepConfigger(config);
    if (csc == null) {
        log.error("Step config create error");
        return;
    }

    if (jobFlowId == null) {
        runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME)
                .withBootstrapActions(
                        new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)),
                        new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI)
                                        .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")),
                        new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction(
                                new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path())))
                .withInstances(setupJobFlowInstancesConfig());
        if (!isEmpty(emrProperties.getLogUri())) {
            runJobFlowRequest.setLogUri(emrProperties.getLogUri());
        }

        List<StepConfig> stepConfigs = new ArrayList<StepConfig>();
        if (emrProperties.isDebug()) {
            StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME)
                    .withActionOnFailure(ACTION_ON_TERMINATE)
                    .withHadoopJarStep(new StepFactory().newEnableDebuggingStep());
            stepConfigs.add(enableDebugging);
        }

        for (StepConfig sc : csc.createStepConfig(config)) {
            stepConfigs.add(sc);
        }
        runJobFlowRequest.setSteps(stepConfigs);

        try {
            RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest);
            jobFlowId = result.getJobFlowId();
            checkDate = new Date();
        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    } else {
        AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId)
                .withSteps(csc.createStepConfig(config));
        emr.addJobFlowSteps(addJobFlowStepsRequest);
    }

    running = true;
    try {
        config.setJobFlowId(jobFlowId);
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }

    int stepSize = 0;
    String stepStatus = JobUtils.STEP_STATUS_PENDING;
    while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) {
        if (sleep()) {
            break;
        }

        DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest()
                .withJobFlowIds(jobFlowId);
        DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest);
        if (describeJobFlowsResult.getJobFlows().size() != 1) {
            break;
        }

        JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0);
        JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances();
        masterPublicDnsName = instancesDetail.getMasterPublicDnsName();
        if (isEmpty(config.getMasterPublicDnsName())) {
            try {
                config.setMasterPublicDnsName(masterPublicDnsName);
                QueueUtils.updateQueue(config);
            } catch (IOException e) {
                e.printStackTrace();
                log.error(e);
            }
        }

        stepSize = jobFlowDetail.getSteps().size();
        for (StepDetail stepDetail : jobFlowDetail.getSteps()) {
            if (stepDetail.getStepConfig().getName().equals(config.getName())) {
                stepStatus = stepDetail.getExecutionStatusDetail().getState();
                break;
            }
        }
    }

    if (config.isDeleteOnExit()) {
        if (config.getJobType() == Config.JOB_TYPE_STREAMING) {
            S3Utils.delete(s3, config.getArgMap().get("mapper"));
            S3Utils.delete(s3, config.getArgMap().get("reducer"));
        } else {
            S3Utils.delete(s3, config.getRun());
        }
    }

    // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV)
    if (stepSize >= 255) {
        instanceTerminate();
    }

    running = false;

    if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) {
        config.setStatus(Config.JOB_STATUS_COMPLETE);
    } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) {
        config.setStatus(Config.JOB_STATUS_ERROR);
    } else if (terminated) {
        config.setStatus(Config.JOB_STATUS_CANCEL);
    }

    try {
        QueueUtils.updateQueue(config);
    } catch (IOException e) {
        e.printStackTrace();
        log.error(e);
    }
}

From source file:org.pentaho.amazon.client.impl.EmrClientImpl.java

License:Apache License

/**
 * Configure a bootstrap action object, given its name, path and arguments.
 *
 * @param path - path for the bootstrap action program in S3
 * @param name - name of the bootstrap action
 * @param args - arguments for the bootstrap action
 * @return configuration data object for one bootstrap action
 *//*from ww  w. jav a2  s  . c  o m*/
private static BootstrapActionConfig configureBootstrapAction(String path, String name, List<String> args) {

    ScriptBootstrapActionConfig scriptBootstrapActionConfig = new ScriptBootstrapActionConfig();
    BootstrapActionConfig bootstrapActionConfig = new BootstrapActionConfig();
    scriptBootstrapActionConfig.setPath(path);
    scriptBootstrapActionConfig.setArgs(args);
    bootstrapActionConfig.setName(name);
    bootstrapActionConfig.setScriptBootstrapAction(scriptBootstrapActionConfig);

    return bootstrapActionConfig;
}

From source file:org.pentaho.amazon.client.impl.EmrClientImpl.java

License:Apache License

/**
 * Configure a bootstrap action object, given its name, path and arguments.
 *
 * @param path - path for the bootstrap action program in S3
 * @param name - name of the bootstrap action
 * @param args - arguments for the bootstrap action
 * @return configuration data object for one bootstrap action
 *///from  www  .  j  av  a  2  s.c  om
private static BootstrapActionConfig createBootstrapAction(String path, String name, List<String> args) {

    ScriptBootstrapActionConfig scriptBootstrapActionConfig = new ScriptBootstrapActionConfig();
    BootstrapActionConfig bootstrapActionConfig = new BootstrapActionConfig();
    if (!path.isEmpty()) {
        scriptBootstrapActionConfig.setPath(path);
        scriptBootstrapActionConfig.setArgs(args);
    }
    bootstrapActionConfig.setName(name);
    bootstrapActionConfig.setScriptBootstrapAction(scriptBootstrapActionConfig);

    return bootstrapActionConfig;
}

From source file:org.pentaho.amazon.hive.job.AmazonHiveJobExecutor.java

License:Apache License

/**
 * Configure a bootstrap action object, given its name, path and arguments.
 * /*from w  w  w . j a v a  2s.  co m*/
 * @param path
 *          - path for the bootstrap action program in S3
 * @param name
 *          - name of the bootstrap action
 * @param args
 *          - arguments for the bootstrap action
 * @return configuration data object for one bootstrap action
 * 
 */
BootstrapActionConfig ConfigureBootstrapAction(String path, String name, List<String> args) {

    ScriptBootstrapActionConfig scriptBootstrapActionConfig = new ScriptBootstrapActionConfig();
    BootstrapActionConfig bootstrapActionConfig = new BootstrapActionConfig();
    scriptBootstrapActionConfig.setPath(path);
    scriptBootstrapActionConfig.setArgs(args);
    bootstrapActionConfig.setName(name);
    bootstrapActionConfig.setScriptBootstrapAction(scriptBootstrapActionConfig);

    return bootstrapActionConfig;
}