List of usage examples for com.amazonaws.services.elasticmapreduce.model JobFlowInstancesConfig setKeepJobFlowAliveWhenNoSteps
public void setKeepJobFlowAliveWhenNoSteps(Boolean keepJobFlowAliveWhenNoSteps)
Specifies whether the cluster should remain available after completing all steps.
From source file:datameer.awstasks.aws.emr.EmrCluster.java
License:Apache License
public synchronized void startup() throws InterruptedException { checkConnection(false);/*from ww w . j a v a 2 s . c o m*/ _clusterState = ClusterState.STARTING; boolean successful = false; try { EmrSettings settings = getSettings(); if (settings.getPrivateKeyName() == null) { throw new NullPointerException( "privateKeyName must not be null please configure settings properly"); } LOG.info("Starting job flow '" + getName() + "' ..."); if (!getRunningJobFlowDetailsByName(getName()).isEmpty()) { throw new IllegalStateException("Job flow with name '" + getName() + "' already running."); } boolean keepAlive = true; JobFlowInstancesConfig jobConfig = new JobFlowInstancesConfig(); jobConfig.setHadoopVersion(_settings.getHadoopVersion()); jobConfig.setMasterInstanceType(settings.getMasterInstanceType().getId()); jobConfig.setSlaveInstanceType(settings.getNodeInstanceType().getId()); jobConfig.setInstanceCount(settings.getInstanceCount()); jobConfig.setEc2KeyName(settings.getPrivateKeyName()); jobConfig.setPlacement(new PlacementType()); jobConfig.setKeepJobFlowAliveWhenNoSteps(keepAlive); final RunJobFlowRequest startRequest = new RunJobFlowRequest(); startRequest.setLogUri("s3n://" + settings.getS3Bucket() + settings.getS3LogPath()); startRequest.setInstances(jobConfig); startRequest.setName(getName()); startRequest.setAdditionalInfo(_settings.getAdditionalStartInfo()); startRequest.setBootstrapActions(_settings.getBootstrapActions()); if (settings.isDebugEnabled()) { startRequest.withSteps(DEBUG_STEP); } RunJobFlowResult startResponse = _emrWebService.runJobFlow(startRequest); _jobFlowId = startResponse.getJobFlowId(); waitUntilClusterStarted(_jobFlowId); LOG.info("elastic cluster '" + getName() + "/" + _jobFlowId + "' started, master-host is " + _masterHost); successful = true; } finally { if (successful) { _clusterState = ClusterState.CONNECTED; } else { _clusterState = ClusterState.UNCONNECTED; _jobFlowId = null; } } }
From source file:org.finra.dm.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Create the job flow instance configuration which contains all the job flow configuration details. * * @param emrClusterDefinition the EMR cluster definition that contains all the EMR parameters. * * @return the job flow instance configuration. *//*from w w w . j av a2 s. com*/ private JobFlowInstancesConfig getJobFlowInstancesConfig(EmrClusterDefinition emrClusterDefinition) { // Create a new job flow instance config object JobFlowInstancesConfig jobFlowInstancesConfig = new JobFlowInstancesConfig(); // Add the DM EMR support security group as additional group to master node. String additionalSecurityGroup = configurationHelper .getProperty(ConfigurationValue.EMR_DM_SUPPORT_SECURITY_GROUP); if (StringUtils.isNotBlank(additionalSecurityGroup)) { List<String> additionalSecurityGroups = new ArrayList<>(); additionalSecurityGroups.add(additionalSecurityGroup); jobFlowInstancesConfig.setAdditionalMasterSecurityGroups(additionalSecurityGroups); } // Fill-in the ssh key if (StringUtils.isNotBlank(emrClusterDefinition.getSshKeyPairName())) { jobFlowInstancesConfig.setEc2KeyName(emrClusterDefinition.getSshKeyPairName()); } // Fill-in subnet id if (StringUtils.isNotBlank(emrClusterDefinition.getSubnetId())) { jobFlowInstancesConfig.setEc2SubnetId(emrClusterDefinition.getSubnetId()); } // Fill in instance groups jobFlowInstancesConfig.setInstanceGroups(getInstanceGroupConfig(emrClusterDefinition)); // Check for optional parameters and then fill-in // Keep Alive Cluster flag if (emrClusterDefinition.isKeepAlive() != null) { jobFlowInstancesConfig.setKeepJobFlowAliveWhenNoSteps(emrClusterDefinition.isKeepAlive()); } // Termination protection flag if (emrClusterDefinition.isTerminationProtection() != null) { jobFlowInstancesConfig.setTerminationProtected(emrClusterDefinition.isTerminationProtection()); } // Setting the hadoop version if (StringUtils.isNotBlank(emrClusterDefinition.getHadoopVersion())) { jobFlowInstancesConfig.setHadoopVersion(emrClusterDefinition.getHadoopVersion()); } // Return the object return jobFlowInstancesConfig; }
From source file:org.finra.herd.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Creates the job flow instance configuration containing specification of the number and type of Amazon EC2 instances. * * @param emrClusterDefinition the EMR cluster definition that contains all the EMR parameters * * @return the job flow instance configuration *//* w ww. j a va2 s. c om*/ private JobFlowInstancesConfig getJobFlowInstancesConfig(EmrClusterDefinition emrClusterDefinition) { // Create a new job flow instances configuration object. JobFlowInstancesConfig jobFlowInstancesConfig = new JobFlowInstancesConfig(); // Set up master/slave security group jobFlowInstancesConfig.setEmrManagedMasterSecurityGroup(emrClusterDefinition.getMasterSecurityGroup()); jobFlowInstancesConfig.setEmrManagedSlaveSecurityGroup(emrClusterDefinition.getSlaveSecurityGroup()); // Add additional security groups to master nodes. jobFlowInstancesConfig .setAdditionalMasterSecurityGroups(emrClusterDefinition.getAdditionalMasterSecurityGroups()); // Add additional security groups to slave nodes. jobFlowInstancesConfig .setAdditionalSlaveSecurityGroups(emrClusterDefinition.getAdditionalSlaveSecurityGroups()); // Fill-in the ssh key. if (StringUtils.isNotBlank(emrClusterDefinition.getSshKeyPairName())) { jobFlowInstancesConfig.setEc2KeyName(emrClusterDefinition.getSshKeyPairName()); } // Fill in configuration for the instance groups in a cluster. jobFlowInstancesConfig .setInstanceGroups(getInstanceGroupConfigs(emrClusterDefinition.getInstanceDefinitions())); // Fill in instance fleet configuration. jobFlowInstancesConfig.setInstanceFleets(getInstanceFleets(emrClusterDefinition.getInstanceFleets())); // Fill-in subnet id. if (StringUtils.isNotBlank(emrClusterDefinition.getSubnetId())) { // Use collection of subnet IDs when instance fleet configuration is specified. Otherwise, we expect a single EC2 subnet ID to be passed here. if (CollectionUtils.isNotEmpty(jobFlowInstancesConfig.getInstanceFleets())) { jobFlowInstancesConfig .setEc2SubnetIds(herdStringHelper.splitAndTrim(emrClusterDefinition.getSubnetId(), ",")); } else { jobFlowInstancesConfig.setEc2SubnetId(emrClusterDefinition.getSubnetId()); } } // Fill in optional keep alive flag. if (emrClusterDefinition.isKeepAlive() != null) { jobFlowInstancesConfig.setKeepJobFlowAliveWhenNoSteps(emrClusterDefinition.isKeepAlive()); } // Fill in optional termination protection flag. if (emrClusterDefinition.isTerminationProtection() != null) { jobFlowInstancesConfig.setTerminationProtected(emrClusterDefinition.isTerminationProtection()); } // Fill in optional Hadoop version flag. if (StringUtils.isNotBlank(emrClusterDefinition.getHadoopVersion())) { jobFlowInstancesConfig.setHadoopVersion(emrClusterDefinition.getHadoopVersion()); } // Return the object. return jobFlowInstancesConfig; }
From source file:org.pentaho.amazon.client.impl.EmrClientImpl.java
License:Apache License
private JobFlowInstancesConfig initEC2Instance(Integer numInsts, String masterInstanceType, String slaveInstanceType) { JobFlowInstancesConfig instances = new JobFlowInstancesConfig(); instances.setInstanceCount(numInsts); instances.setMasterInstanceType(masterInstanceType); instances.setSlaveInstanceType(slaveInstanceType); instances.setKeepJobFlowAliveWhenNoSteps(alive); return instances; }
From source file:org.pentaho.amazon.hive.job.AmazonHiveJobExecutor.java
License:Apache License
/** * Prepare to create a EMR job flow./*from w ww . jav a 2 s . c o m*/ * * @return RunJobFlowRequest The object to request an EMR job flow */ public RunJobFlowRequest createJobFlow() { // Create a RunJobFlowRequest object, set a name for the job flow. RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(); runJobFlowRequest.setName(hadoopJobName); // Set a log URL. String logUrl = stagingDir; if (stagingDir.indexOf("@s3") > 0) { //$NON-NLS-1$ logUrl = S3FileProvider.SCHEME + "://" + stagingDir.substring(stagingDir.indexOf("@s3") + 4); //$NON-NLS-1$ } runJobFlowRequest.setLogUri(logUrl); // Determine the instances for Hadoop cluster. String numInstancesS = environmentSubstitute(numInstances); int numInsts = 2; try { numInsts = Integer.parseInt(numInstancesS); } catch (NumberFormatException e) { logError(BaseMessages.getString(PKG, "AmazonElasticMapReduceJobExecutor.InstanceNumber.Error", //$NON-NLS-1$ numInstancesS)); } JobFlowInstancesConfig instances = new JobFlowInstancesConfig(); instances.setInstanceCount(numInsts); instances.setMasterInstanceType(getInstanceType(masterInstanceType)); instances.setSlaveInstanceType(getInstanceType(slaveInstanceType)); instances.setHadoopVersion("0.20"); //$NON-NLS-1$ instances.setKeepJobFlowAliveWhenNoSteps(alive); runJobFlowRequest.setInstances(instances); // Set bootstrap actions. runJobFlowRequest.setBootstrapActions(ConfigBootstrapActions()); // Create an EMR step to setup Hive. String args = "s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions 0.7 --install-hive"; //$NON-NLS-1$ List<StepConfig> steps = ConfigHadoopJarStep("Setup Hive", jarUrl, args); //$NON-NLS-1$ runJobFlowRequest.setSteps(steps); return runJobFlowRequest; }