List of usage examples for com.amazonaws.services.elasticmapreduce.model JobFlowInstancesConfig setEc2SubnetId
public void setEc2SubnetId(String ec2SubnetId)
Applies to clusters that use the uniform instance group configuration.
From source file:com.aegeus.aws.ElasticMapReduceService.java
License:Apache License
/** * Create a new EMR Cluster over Hadoop 2.4.0 *///from w w w.ja v a 2s. com public void createCluster() { JobFlowInstancesConfig instances = new JobFlowInstancesConfig() .withInstanceCount((int) config.getInstanceCount()).withMasterInstanceType(config.getMasterType()) .withSlaveInstanceType(config.getSlaveType()); if (Strings.isNullOrEmpty(config.getKeyName())) { instances.setEc2KeyName(config.getKeyName()); } if (!Strings.isNullOrEmpty(config.getSubnetId())) { instances.setEc2SubnetId(config.getSubnetId()); } else { instances.setPlacement(new PlacementType(config.getPlace())); } ScriptBootstrapActionConfig installEsConfig = new ScriptBootstrapActionConfig() .withPath("s3://support.elasticmapreduce/bootstrap-actions/other/elasticsearch_install.rb"); BootstrapActionConfig installEs = new BootstrapActionConfig("Elasticsearch Install", installEsConfig); RunJobFlowRequest request = new RunJobFlowRequest().withName(config.getName()).withReleaseLabel("emr-4.1.0") .withServiceRole("Default_AWS_Role").withJobFlowRole("Default_AWS_Role") .withBootstrapActions(installEs).withInstances(instances); if (!Strings.isNullOrEmpty(config.getLogBucket())) { request.setLogUri(config.getLogBucket()); } RunJobFlowResult result = emr.runJobFlow(request); clusterId = result.getJobFlowId(); }
From source file:org.finra.dm.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Create the job flow instance configuration which contains all the job flow configuration details. * * @param emrClusterDefinition the EMR cluster definition that contains all the EMR parameters. * * @return the job flow instance configuration. *///from w w w . j a v a 2 s . com private JobFlowInstancesConfig getJobFlowInstancesConfig(EmrClusterDefinition emrClusterDefinition) { // Create a new job flow instance config object JobFlowInstancesConfig jobFlowInstancesConfig = new JobFlowInstancesConfig(); // Add the DM EMR support security group as additional group to master node. String additionalSecurityGroup = configurationHelper .getProperty(ConfigurationValue.EMR_DM_SUPPORT_SECURITY_GROUP); if (StringUtils.isNotBlank(additionalSecurityGroup)) { List<String> additionalSecurityGroups = new ArrayList<>(); additionalSecurityGroups.add(additionalSecurityGroup); jobFlowInstancesConfig.setAdditionalMasterSecurityGroups(additionalSecurityGroups); } // Fill-in the ssh key if (StringUtils.isNotBlank(emrClusterDefinition.getSshKeyPairName())) { jobFlowInstancesConfig.setEc2KeyName(emrClusterDefinition.getSshKeyPairName()); } // Fill-in subnet id if (StringUtils.isNotBlank(emrClusterDefinition.getSubnetId())) { jobFlowInstancesConfig.setEc2SubnetId(emrClusterDefinition.getSubnetId()); } // Fill in instance groups jobFlowInstancesConfig.setInstanceGroups(getInstanceGroupConfig(emrClusterDefinition)); // Check for optional parameters and then fill-in // Keep Alive Cluster flag if (emrClusterDefinition.isKeepAlive() != null) { jobFlowInstancesConfig.setKeepJobFlowAliveWhenNoSteps(emrClusterDefinition.isKeepAlive()); } // Termination protection flag if (emrClusterDefinition.isTerminationProtection() != null) { jobFlowInstancesConfig.setTerminationProtected(emrClusterDefinition.isTerminationProtection()); } // Setting the hadoop version if (StringUtils.isNotBlank(emrClusterDefinition.getHadoopVersion())) { jobFlowInstancesConfig.setHadoopVersion(emrClusterDefinition.getHadoopVersion()); } // Return the object return jobFlowInstancesConfig; }
From source file:org.finra.herd.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Creates the job flow instance configuration containing specification of the number and type of Amazon EC2 instances. * * @param emrClusterDefinition the EMR cluster definition that contains all the EMR parameters * * @return the job flow instance configuration *//* w w w . j a v a2 s. c o m*/ private JobFlowInstancesConfig getJobFlowInstancesConfig(EmrClusterDefinition emrClusterDefinition) { // Create a new job flow instances configuration object. JobFlowInstancesConfig jobFlowInstancesConfig = new JobFlowInstancesConfig(); // Set up master/slave security group jobFlowInstancesConfig.setEmrManagedMasterSecurityGroup(emrClusterDefinition.getMasterSecurityGroup()); jobFlowInstancesConfig.setEmrManagedSlaveSecurityGroup(emrClusterDefinition.getSlaveSecurityGroup()); // Add additional security groups to master nodes. jobFlowInstancesConfig .setAdditionalMasterSecurityGroups(emrClusterDefinition.getAdditionalMasterSecurityGroups()); // Add additional security groups to slave nodes. jobFlowInstancesConfig .setAdditionalSlaveSecurityGroups(emrClusterDefinition.getAdditionalSlaveSecurityGroups()); // Fill-in the ssh key. if (StringUtils.isNotBlank(emrClusterDefinition.getSshKeyPairName())) { jobFlowInstancesConfig.setEc2KeyName(emrClusterDefinition.getSshKeyPairName()); } // Fill in configuration for the instance groups in a cluster. jobFlowInstancesConfig .setInstanceGroups(getInstanceGroupConfigs(emrClusterDefinition.getInstanceDefinitions())); // Fill in instance fleet configuration. jobFlowInstancesConfig.setInstanceFleets(getInstanceFleets(emrClusterDefinition.getInstanceFleets())); // Fill-in subnet id. if (StringUtils.isNotBlank(emrClusterDefinition.getSubnetId())) { // Use collection of subnet IDs when instance fleet configuration is specified. Otherwise, we expect a single EC2 subnet ID to be passed here. if (CollectionUtils.isNotEmpty(jobFlowInstancesConfig.getInstanceFleets())) { jobFlowInstancesConfig .setEc2SubnetIds(herdStringHelper.splitAndTrim(emrClusterDefinition.getSubnetId(), ",")); } else { jobFlowInstancesConfig.setEc2SubnetId(emrClusterDefinition.getSubnetId()); } } // Fill in optional keep alive flag. if (emrClusterDefinition.isKeepAlive() != null) { jobFlowInstancesConfig.setKeepJobFlowAliveWhenNoSteps(emrClusterDefinition.isKeepAlive()); } // Fill in optional termination protection flag. if (emrClusterDefinition.isTerminationProtection() != null) { jobFlowInstancesConfig.setTerminationProtected(emrClusterDefinition.isTerminationProtection()); } // Fill in optional Hadoop version flag. if (StringUtils.isNotBlank(emrClusterDefinition.getHadoopVersion())) { jobFlowInstancesConfig.setHadoopVersion(emrClusterDefinition.getHadoopVersion()); } // Return the object. return jobFlowInstancesConfig; }