Example usage for com.amazonaws.services.elasticmapreduce AmazonElasticMapReduceClientBuilder standard

Introduction

In this page you can find the example usage for com.amazonaws.services.elasticmapreduce AmazonElasticMapReduceClientBuilder standard.

Prototype

public static AmazonElasticMapReduceClientBuilder standard() 

Source Link

Usage

From source file:org.deeplearning4j.legacyExamples.EmrSparkExample.java

License:Apache License

public void entryPoint(String[] args) {
    JCommander jcmdr = new JCommander(this);
    try {/* w w w.  ja  v  a  2s .  com*/
        jcmdr.parse(args);
    } catch (ParameterException e) {
        jcmdr.usage();
        try {
            Thread.sleep(500);
        } catch (Exception e2) {
        }
        throw e;
    }

    AmazonElasticMapReduceClientBuilder builder = AmazonElasticMapReduceClientBuilder.standard();
    builder.withRegion(region);
    builder.withCredentials(getCredentialsProvider());

    AmazonElasticMapReduce emr = builder.build();

    List<StepConfig> steps = new ArrayList<>();

    if (upload) {
        log.info("uploading uber jar");

        AmazonS3ClientBuilder s3builder = AmazonS3ClientBuilder.standard();
        s3builder.withRegion(region);
        s3builder.withCredentials(getCredentialsProvider());
        AmazonS3 s3Client = s3builder.build();

        if (!s3Client.doesBucketExist(bucketName)) {
            s3Client.createBucket(bucketName);
        }

        File uberJarFile = new File(uberJar);

        s3Client.putObject(new PutObjectRequest(bucketName, uberJarFile.getName(), uberJarFile));
    }

    if (debug) {
        log.info("enable debug");

        StepFactory stepFactory = new StepFactory(builder.getRegion() + ".elasticmapreduce");
        StepConfig enableDebugging = new StepConfig().withName("Enable Debugging")
                .withActionOnFailure(ActionOnFailure.TERMINATE_JOB_FLOW)
                .withHadoopJarStep(stepFactory.newEnableDebuggingStep());
        steps.add(enableDebugging);
    }

    if (execute) {
        log.info("execute spark step");

        HadoopJarStepConfig sparkStepConf = new HadoopJarStepConfig();
        sparkStepConf.withJar("command-runner.jar");
        sparkStepConf.withArgs("spark-submit", "--deploy-mode", "cluster", "--class", className,
                getS3UberJarUrl(), "-useSparkLocal", "false");

        ActionOnFailure action = ActionOnFailure.TERMINATE_JOB_FLOW;

        if (keepAlive) {
            action = ActionOnFailure.CONTINUE;
        }

        StepConfig sparkStep = new StepConfig().withName("Spark Step").withActionOnFailure(action)
                .withHadoopJarStep(sparkStepConf);
        steps.add(sparkStep);
    }

    log.info("create spark cluster");

    Application sparkApp = new Application().withName("Spark");

    // service and job flow role will be created automatically when
    // launching cluster in aws console, better do that first or create
    // manually

    RunJobFlowRequest request = new RunJobFlowRequest().withName("Spark Cluster").withSteps(steps)
            .withServiceRole("EMR_DefaultRole").withJobFlowRole("EMR_EC2_DefaultRole")
            .withApplications(sparkApp).withReleaseLabel(emrVersion).withLogUri(getS3BucketLogsUrl())
            .withInstances(new JobFlowInstancesConfig().withEc2KeyName("spark").withInstanceCount(instanceCount)
                    .withKeepJobFlowAliveWhenNoSteps(keepAlive).withMasterInstanceType(instanceType)
                    .withSlaveInstanceType(instanceType));

    RunJobFlowResult result = emr.runJobFlow(request);

    log.info(result.toString());

    log.info("done");
}

---

From source file:org.finra.herd.dao.AwsClientFactory.java

License:Apache License

/**
 * Creates a client for accessing Amazon EMR service.
 *
 * @param awsParamsDto the AWS related parameters DTO that includes optional AWS credentials and proxy information
 *
 * @return the Amazon EMR client/*from w  w w  .jav a 2s .c  o  m*/
 */
@Cacheable(DaoSpringModuleConfig.HERD_CACHE_NAME)
public AmazonElasticMapReduce getEmrClient(AwsParamsDto awsParamsDto) {
    // Get client configuration.
    ClientConfiguration clientConfiguration = awsHelper.getClientConfiguration(awsParamsDto);

    // If specified, use the AWS credentials passed in.
    if (StringUtils.isNotBlank(awsParamsDto.getAwsAccessKeyId())) {
        return AmazonElasticMapReduceClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(
                        new BasicSessionCredentials(awsParamsDto.getAwsAccessKeyId(),
                                awsParamsDto.getAwsSecretKey(), awsParamsDto.getSessionToken())))
                .withClientConfiguration(clientConfiguration).withRegion(awsParamsDto.getAwsRegionName())
                .build();
    }
    // Otherwise, use the default AWS credentials provider chain.
    else {
        return AmazonElasticMapReduceClientBuilder.standard().withClientConfiguration(clientConfiguration)
                .withRegion(awsParamsDto.getAwsRegionName()).build();
    }
}

---

From source file:org.pentaho.amazon.client.impl.EmrClientFactory.java

License:Apache License

@Override
public EmrClient createClient(String accessKey, String secretKey, String region) {
    AmazonClientCredentials clientCredentials = new AmazonClientCredentials(accessKey, secretKey, region);

    AmazonElasticMapReduce awsEmrClient = AmazonElasticMapReduceClientBuilder.standard()
            .withRegion(clientCredentials.getRegion())
            .withCredentials(new AWSStaticCredentialsProvider(clientCredentials.getAWSCredentials())).build();

    EmrClient emrClient = new EmrClientImpl(awsEmrClient);

    return emrClient;
}

---