List of usage examples for com.amazonaws.services.elasticmapreduce.model RunJobFlowRequest setApplications
public void setApplications(java.util.Collection<Application> applications)
Applies to Amazon EMR releases 4.0 and later.
From source file:org.finra.dm.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Create the run job flow request object. * * @param emrClusterDefinition the EMR definition name value. * @param clusterName the EMR cluster name. * * @return run job flow request for the given configuration. *///from w w w . j a v a2s . c o m private RunJobFlowRequest getRunJobFlowRequest(String clusterName, EmrClusterDefinition emrClusterDefinition) { // Create the object RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); // Set release label if (StringUtils.isNotBlank(emrClusterDefinition.getReleaseLabel())) { runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel()); } // Set list of Applications List<EmrClusterDefinitionApplication> emrClusterDefinitionApplications = emrClusterDefinition .getApplications(); if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) { runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); } // Set list of Configurations List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations = emrClusterDefinition .getConfigurations(); if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) { runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); } // Set the log bucket if specified if (StringUtils.isNotBlank(emrClusterDefinition.getLogBucket())) { runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket()); } // Set the visible to all flag if (emrClusterDefinition.isVisibleToAll() != null) { runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll()); } // Set the IAM profile for the nodes if (StringUtils.isNotBlank(emrClusterDefinition.getEc2NodeIamProfileName())) { runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName()); } else { runJobFlowRequest.setJobFlowRole(dmStringHelper .getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME)); } // Set the IAM profile for the service if (StringUtils.isNotBlank(emrClusterDefinition.getServiceIamRole())) { runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole()); } else { runJobFlowRequest.setServiceRole(dmStringHelper .getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME)); } // Set the AMI version if specified if (StringUtils.isNotBlank(emrClusterDefinition.getAmiVersion())) { runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion()); } // Set the additionalInfo if specified if (StringUtils.isNotBlank(emrClusterDefinition.getAdditionalInfo())) { runJobFlowRequest.setAdditionalInfo(emrClusterDefinition.getAdditionalInfo()); } // Set the bootstrap actions if (!getBootstrapActionConfigList(emrClusterDefinition).isEmpty()) { runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition)); } // Set the app installation steps if (!getStepConfig(emrClusterDefinition).isEmpty()) { runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); } // Set the tags if (!getEmrTags(emrClusterDefinition).isEmpty()) { runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition)); } // Assign supported products as applicable if (StringUtils.isNotBlank(emrClusterDefinition.getSupportedProduct())) { List<String> supportedProducts = new ArrayList<>(); supportedProducts.add(emrClusterDefinition.getSupportedProduct()); runJobFlowRequest.setSupportedProducts(supportedProducts); } // Return the object return runJobFlowRequest; }
From source file:org.finra.herd.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Create the run job flow request object. * * @param emrClusterDefinition the EMR definition name value * @param clusterName the EMR cluster name * * @return the run job flow request for the given configuration *///from w w w .ja v a2 s. c o m private RunJobFlowRequest getRunJobFlowRequest(String clusterName, EmrClusterDefinition emrClusterDefinition) { // Create the object RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); // Set release label if (StringUtils.isNotBlank(emrClusterDefinition.getReleaseLabel())) { runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel()); } // Set list of Applications List<EmrClusterDefinitionApplication> emrClusterDefinitionApplications = emrClusterDefinition .getApplications(); if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) { runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); } // Set list of Configurations List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations = emrClusterDefinition .getConfigurations(); if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) { runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); } // Set the log bucket if specified if (StringUtils.isNotBlank(emrClusterDefinition.getLogBucket())) { runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket()); } // Set the visible to all flag if (emrClusterDefinition.isVisibleToAll() != null) { runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll()); } // Set the IAM profile for the nodes if (StringUtils.isNotBlank(emrClusterDefinition.getEc2NodeIamProfileName())) { runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName()); } else { runJobFlowRequest.setJobFlowRole(herdStringHelper .getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME)); } // Set the IAM profile for the service if (StringUtils.isNotBlank(emrClusterDefinition.getServiceIamRole())) { runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole()); } else { runJobFlowRequest.setServiceRole(herdStringHelper .getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME)); } // Set the AMI version if specified if (StringUtils.isNotBlank(emrClusterDefinition.getAmiVersion())) { runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion()); } // Set the additionalInfo if specified if (StringUtils.isNotBlank(emrClusterDefinition.getAdditionalInfo())) { runJobFlowRequest.setAdditionalInfo(emrClusterDefinition.getAdditionalInfo()); } // Set the bootstrap actions List<BootstrapActionConfig> bootstrapActionConfigList = getBootstrapActionConfigList(emrClusterDefinition); if (!bootstrapActionConfigList.isEmpty()) { runJobFlowRequest.setBootstrapActions(bootstrapActionConfigList); } // Set the app installation steps runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); // Set the tags runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition)); // Assign supported products as applicable if (StringUtils.isNotBlank(emrClusterDefinition.getSupportedProduct())) { List<String> supportedProducts = new ArrayList<>(); supportedProducts.add(emrClusterDefinition.getSupportedProduct()); runJobFlowRequest.setSupportedProducts(supportedProducts); } // Assign security configuration. if (StringUtils.isNotBlank(emrClusterDefinition.getSecurityConfiguration())) { runJobFlowRequest.setSecurityConfiguration(emrClusterDefinition.getSecurityConfiguration()); } // Assign scale down behavior. if (StringUtils.isNotBlank(emrClusterDefinition.getScaleDownBehavior())) { runJobFlowRequest.setScaleDownBehavior(emrClusterDefinition.getScaleDownBehavior()); } // Assign Kerberos attributes. runJobFlowRequest .setKerberosAttributes(getKerberosAttributes(emrClusterDefinition.getKerberosAttributes())); // Return the object return runJobFlowRequest; }
From source file:org.pentaho.amazon.client.impl.EmrClientImpl.java
License:Apache License
@VisibleForTesting RunJobFlowRequest initEmrCluster(String stagingS3FileUrl, String stagingS3BucketUrl, String stepType, String mainClass, String bootstrapActions, AbstractAmazonJobEntry jobEntry) { RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(); runJobFlowRequest.setName(jobEntry.getHadoopJobName()); runJobFlowRequest.setReleaseLabel(jobEntry.getEmrRelease()); runJobFlowRequest.setLogUri(stagingS3BucketUrl); JobFlowInstancesConfig instances = initEC2Instance(Integer.parseInt(jobEntry.getNumInstances()), jobEntry.getMasterInstanceType(), jobEntry.getSlaveInstanceType()); runJobFlowRequest.setInstances(instances); List<StepConfig> steps = initSteps(stagingS3FileUrl, stepType, mainClass, jobEntry); if (steps.size() > 0) { runJobFlowRequest.setSteps(steps); }//from w w w. ja v a2 s .co m if (stepType.equals(STEP_HIVE)) { List<Application> applications = initApplications(); if (applications.size() > 0) { runJobFlowRequest.setApplications(applications); } List<BootstrapActionConfig> stepBootstrapActions = initBootstrapActions(bootstrapActions); if (stepBootstrapActions != null && stepBootstrapActions.size() > 0) { runJobFlowRequest.setBootstrapActions(stepBootstrapActions); } } String ec2Role = jobEntry.getEc2Role(); if (ec2Role == null || ec2Role.trim().isEmpty()) { runJobFlowRequest.setJobFlowRole(EMR_EC2_DEFAULT_ROLE); } else { runJobFlowRequest.setJobFlowRole(ec2Role); } String emrRole = jobEntry.getEmrRole(); if (emrRole == null || emrRole.trim().isEmpty()) { runJobFlowRequest.setServiceRole(EMR_EFAULT_ROLE); } else { runJobFlowRequest.setServiceRole(emrRole); } runJobFlowRequest.setVisibleToAllUsers(true); return runJobFlowRequest; }