List of usage examples for com.amazonaws.services.elasticmapreduce.model ClusterSummary getStatus
public ClusterStatus getStatus()
The details about the current status of the cluster.
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * Gets details of an existing EMR Cluster. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key * @param emrClusterId the cluster id of the cluster to get details * @param emrStepId the step id of the step to get details * @param verbose parameter for whether to return detailed information * @param retrieveOozieJobs parameter for whether to retrieve oozie job information * * @return the EMR Cluster object with details. * @throws Exception if an error occurred while getting the cluster. *//* ww w. j a v a2 s .com*/ protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, boolean retrieveOozieJobs) throws Exception { // Perform the request validation. emrHelper.validateEmrClusterKey(emrClusterAlternateKeyDto); // Get the namespace and ensure it exists. NamespaceEntity namespaceEntity = dmDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace()); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = dmDaoHelper.getEmrClusterDefinitionEntity( emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName()); EmrCluster emrCluster = createEmrClusterFromRequest(null, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), null, null, null, null); String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { // Get Cluster status if clusterId is specified if (StringUtils.hasText(emrClusterId)) { Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), emrHelper.getAwsParamsDto()); // Validate that, Cluster exists Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\"."); // Validate that, Cluster name match as specified Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified."); emrCluster.setId(cluster.getId()); emrCluster.setStatus(cluster.getStatus().getState()); } else { ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, emrHelper.getAwsParamsDto()); // Validate that, Cluster exists with the name Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\"."); emrCluster.setId(clusterSummary.getId()); emrCluster.setStatus(clusterSummary.getStatus().getState()); } // Get active step details if (emrHelper.isActiveEmrState(emrCluster.getStatus())) { StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), emrHelper.getAwsParamsDto()); if (stepSummary != null) { EmrStep activeStep; // If verbose get active step details if (verbose) { activeStep = buildEmrStepFromAwsStep(emrDao.getClusterStep(emrCluster.getId(), stepSummary.getId(), emrHelper.getAwsParamsDto()), true); } else { activeStep = buildEmrStepFromAwsStepSummary(stepSummary); } emrCluster.setActiveStep(activeStep); } } // Get requested step details if (StringUtils.hasText(emrStepId)) { Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), emrHelper.getAwsParamsDto()); emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose)); } // Get oozie job details if requested. if (retrieveOozieJobs && (emrCluster.getStatus().equalsIgnoreCase("RUNNING") || emrCluster.getStatus().equalsIgnoreCase("WAITING"))) { emrCluster.setOozieWorkflowJobs(retrieveOozieJobs(emrCluster.getId())); } } catch (AmazonServiceException ex) { handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\"."); } return emrCluster; }
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * Get the cluster in RUNNING or WAITING status. * * @param namespace namespace/* w w w . j a v a2s . c om*/ * @param emrClusterDefinitionName emrClusterDefinitionName * @param emrClusterName emrClusterName * * @return ClusterSummary */ private ClusterSummary getRunningOrWaitingEmrCluster(String namespace, String emrClusterDefinitionName, String emrClusterName) { // Get the namespace and ensure it exists. NamespaceEntity namespaceEntity = dmDaoHelper.getNamespaceEntity(namespace); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = dmDaoHelper.getEmrClusterDefinitionEntity(namespace, emrClusterDefinitionName); String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterName); // Look up cluster ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, emrHelper.getAwsParamsDto()); // We can only run oozie job when the cluster is up (RUNNING or WAITING). Can not submit job otherwise like bootstraping. // Make sure that cluster exists and is in RUNNING or WAITING state. if (clusterSummary == null || !(clusterSummary.getStatus().getState().equalsIgnoreCase("RUNNING") || clusterSummary.getStatus().getState().equalsIgnoreCase("WAITING"))) { throw new ObjectNotFoundException(String.format( "Either the cluster \"%s\" does not exist or not in RUNNING or WAITING state.", clusterName)); } return clusterSummary; }
From source file:org.finra.herd.service.impl.EmrServiceImpl.java
License:Apache License
/** * Gets details of an existing EMR Cluster. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key * @param emrClusterId the cluster id of the cluster to get details * @param emrStepId the step id of the step to get details * @param verbose parameter for whether to return detailed information * @param accountId the optional AWS account that EMR cluster is running in * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets * * @return the EMR Cluster object with details. *///from w w w.j a v a2 s .co m protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, String accountId, Boolean retrieveInstanceFleets) { AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(accountId); // Perform the request validation. validateEmrClusterKey(emrClusterAlternateKeyDto); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper .getEmrClusterDefinitionEntity(new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName())); EmrCluster emrCluster = createEmrClusterFromRequest(null, emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null); String clusterName = emrHelper.buildEmrClusterName(emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { // Get Cluster status if clusterId is specified if (StringUtils.isNotBlank(emrClusterId)) { Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto); // Validate that, Cluster exists Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\"."); // Validate that, Cluster name match as specified Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified."); emrCluster.setId(cluster.getId()); setEmrClusterStatus(emrCluster, cluster.getStatus()); } else { ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto); // Validate that, Cluster exists with the name Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\"."); emrCluster.setId(clusterSummary.getId()); setEmrClusterStatus(emrCluster, clusterSummary.getStatus()); } // Get active step details if (emrHelper.isActiveEmrState(emrCluster.getStatus())) { StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto); if (stepSummary != null) { EmrStep activeStep; // If verbose get active step details if (verbose) { activeStep = buildEmrStepFromAwsStep(stepSummary, true); } else { activeStep = buildEmrStepFromAwsStepSummary(stepSummary); } emrCluster.setActiveStep(activeStep); } } // Get requested step details if (StringUtils.isNotBlank(emrStepId)) { Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto); emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose)); } // Get instance fleet if true if (BooleanUtils.isTrue(retrieveInstanceFleets)) { ListInstanceFleetsResult listInstanceFleetsResult = emrDao .getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto); emrCluster.setInstanceFleets( emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult)); } } catch (AmazonServiceException ex) { awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\"."); } return emrCluster; }
From source file:rollsPOC2.util.AWSHelper.java
public static String createOrFindEMRHiveCluster(String clusterName, boolean createWithKeepAlive) throws Exception { String clusterId = null;/*from w ww . j a v a 2s .c om*/ AmazonElasticMapReduce emr = AppServices.getEMRClient(); ClusterSummary clusterSummary = findCluster("Treebeard", emr); if (clusterSummary != null) { clusterId = clusterSummary.getId(); System.err.printf("Cluster found with id %s, status %s\n", clusterId, clusterSummary.getStatus().getState()); } if (clusterSummary != null && clusterSummary.getStatus().getState().startsWith("TERMINAT")) { while (findCluster("Treebeard", emr).getStatus().getState().equals("TERMINATING")) { System.out.println("Waiting for previous cluster to terminate"); Thread.sleep(10000l); } System.out.println("Starting cluster..."); StepFactory stepFactory = new StepFactory(); StepConfig enabledebugging = new StepConfig().withName("Enable debugging") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(stepFactory.newEnableDebuggingStep()); // Possibly redundant with ".withApplications(new Application().withName("Hive"))" // StepConfig installHive = new StepConfig() // .withName("Install Hive") // .withActionOnFailure("TERMINATE_JOB_FLOW") // .withHadoopJarStep(stepFactory.newInstallHiveStep()); RunJobFlowRequest request = new RunJobFlowRequest().withName("Treebeard").withReleaseLabel("emr-4.6.0") .withApplications(new Application().withName("Hive")).withSteps(enabledebugging) .withVisibleToAllUsers(true) .withLogUri("s3://aws-logs-800327301943-us-east-1/elasticmapreduce/") .withServiceRole("EMR_DefaultRole").withJobFlowRole("EMR_EC2_DefaultRole") .withInstances(new JobFlowInstancesConfig().withEc2KeyName("bjss").withInstanceCount(2) .withMasterInstanceType("m3.xlarge").withSlaveInstanceType("m1.large") .withKeepJobFlowAliveWhenNoSteps(createWithKeepAlive)); RunJobFlowResult createClusterResult = emr.runJobFlow(request); clusterId = createClusterResult.getJobFlowId(); System.out.printf("Started cluster with id %s\n", clusterId); } return clusterId; }