List of usage examples for com.amazonaws.services.elasticmapreduce.model ClusterSummary getId
public String getId()
The unique identifier for the cluster.
From source file:org.finra.dm.dao.impl.EmrDaoImpl.java
License:Apache License
/** * Get an Active EMR cluster id by the cluster name. Cluster only in following states are returned: ClusterState.BOOTSTRAPPING, ClusterState.RUNNING, * ClusterState.STARTING, ClusterState.WAITING * * @param awsParams AWS related parameters for access/secret keys and proxy details. * @param clusterName the cluster name value. * * @return the ClusterSummary object./*from w w w .j a v a2s . c o m*/ */ @Override public String getActiveEmrClusterIdByName(String clusterName, AwsParamsDto awsParams) { ClusterSummary clusterSummary = getActiveEmrClusterByName(clusterName, awsParams); return (clusterSummary == null ? null : clusterSummary.getId()); }
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * Gets details of an existing EMR Cluster. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key * @param emrClusterId the cluster id of the cluster to get details * @param emrStepId the step id of the step to get details * @param verbose parameter for whether to return detailed information * @param retrieveOozieJobs parameter for whether to retrieve oozie job information * * @return the EMR Cluster object with details. * @throws Exception if an error occurred while getting the cluster. *//*from w ww. j ava 2 s . com*/ protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, boolean retrieveOozieJobs) throws Exception { // Perform the request validation. emrHelper.validateEmrClusterKey(emrClusterAlternateKeyDto); // Get the namespace and ensure it exists. NamespaceEntity namespaceEntity = dmDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace()); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = dmDaoHelper.getEmrClusterDefinitionEntity( emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName()); EmrCluster emrCluster = createEmrClusterFromRequest(null, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), null, null, null, null); String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { // Get Cluster status if clusterId is specified if (StringUtils.hasText(emrClusterId)) { Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), emrHelper.getAwsParamsDto()); // Validate that, Cluster exists Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\"."); // Validate that, Cluster name match as specified Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified."); emrCluster.setId(cluster.getId()); emrCluster.setStatus(cluster.getStatus().getState()); } else { ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, emrHelper.getAwsParamsDto()); // Validate that, Cluster exists with the name Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\"."); emrCluster.setId(clusterSummary.getId()); emrCluster.setStatus(clusterSummary.getStatus().getState()); } // Get active step details if (emrHelper.isActiveEmrState(emrCluster.getStatus())) { StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), emrHelper.getAwsParamsDto()); if (stepSummary != null) { EmrStep activeStep; // If verbose get active step details if (verbose) { activeStep = buildEmrStepFromAwsStep(emrDao.getClusterStep(emrCluster.getId(), stepSummary.getId(), emrHelper.getAwsParamsDto()), true); } else { activeStep = buildEmrStepFromAwsStepSummary(stepSummary); } emrCluster.setActiveStep(activeStep); } } // Get requested step details if (StringUtils.hasText(emrStepId)) { Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), emrHelper.getAwsParamsDto()); emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose)); } // Get oozie job details if requested. if (retrieveOozieJobs && (emrCluster.getStatus().equalsIgnoreCase("RUNNING") || emrCluster.getStatus().equalsIgnoreCase("WAITING"))) { emrCluster.setOozieWorkflowJobs(retrieveOozieJobs(emrCluster.getId())); } } catch (AmazonServiceException ex) { handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\"."); } return emrCluster; }
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * <p> Creates a new EMR cluster based on the given request if the cluster with the given name does not already exist. If the cluster already exist, returns * the information about the existing cluster. </p> <p> The request must contain: </p> <ul> <li>A namespace and definition name which refer to an existing * EMR cluster definition.</li> <li>A valid cluster name to create.</li> </ul> <p> The request may optionally contain: </p> <ul> <li>A "dry run" flag, which * when set to {@code true}, no calls to AWS will occur, but validations and override will. Defaults to {@code false}.</li> <li>An override parameter, which * when set, overrides the given parameters in the cluster definition before creating the cluster. Defaults to no override. </li> </ul> <p> A successful * response will contain: </p> <ul> <li>The ID of the cluster that was created, or if the cluster already exists, the ID of the cluster that exists. This * field will be {@code null} when dry run flag is {@code true}.</li> <li>The status of the cluster that was created or already exists. The status will * normally be "Starting" on successful creations. This field will be {@code null} when dry run flag is {@code true}</li> <li>The namespace, definition * name, and cluster name of the cluster.</li> <li>The dry run flag, if given in the request.</li> <li>An indicator whether the cluster was created or not. * If the cluster already exists, the cluster will not be created and this flag will be set to {@code false}.</li> <li>The definition which was used to * create the cluster. If any overrides were given, this definition's values will be the values after the override. This field will be {@code null} if the * cluster was not created.</li> </ul> <p> Notes: </p> <ul> <li>At any point of the execution, if there are validation errors, the method will immediately * throw an exception.</li> <li>Even if the validations pass, AWS may still reject the request, which will cause this method to throw an exception.</li> * <li>Dry runs do not make any calls to AWS, therefore AWS may still reject the creation request even when a dry run succeeds.</li> </ul> * * @param request - {@link EmrClusterCreateRequest} The EMR cluster create request * * @return {@link EmrCluster} the created EMR cluster object * @throws Exception when the original EMR cluster definition XML is malformed *//*from w w w.j a v a 2 s . c o m*/ protected EmrCluster createClusterImpl(EmrClusterCreateRequest request) throws Exception { AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDto(); // Extract EMR cluster alternate key from the create request. EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = getEmrClusterAlternateKey(request); // Perform the request validation. emrHelper.validateEmrClusterKey(emrClusterAlternateKeyDto); // Get the namespace and ensure it exists. NamespaceEntity namespaceEntity = dmDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace()); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = dmDaoHelper.getEmrClusterDefinitionEntity( emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName()); // Replace all S3 managed location variables in xml String toReplace = getS3ManagedReplaceString(); String replacedConfigXml = emrClusterDefinitionEntity.getConfiguration().replaceAll(toReplace, emrHelper.getS3StagingLocation()); // Unmarshal definition xml into JAXB object. EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, replacedConfigXml); // Perform override if override is set overrideEmrClusterDefinition(emrClusterDefinition, request.getEmrClusterDefinitionOverride()); // Perform the EMR cluster definition configuration validation. dmHelper.validateEmrClusterDefinitionConfiguration(emrClusterDefinition); // Find best price and update definition emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterDefinition); String clusterId = null; // The cluster ID record. String emrClusterStatus = null; Boolean emrClusterCreated = null; // Was cluster created? // If the dryRun flag is null or false. This is the default option if no flag is given. if (!Boolean.TRUE.equals(request.isDryRun())) { /* * Create the cluster only if the cluster does not already exist. * If the cluster is created, record the newly created cluster ID. * If the cluster already exists, record the existing cluster ID. * If there is any error while attempting to check for existing cluster or create a new one, handle the exception to throw appropriate exception. */ String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto); // If cluster does not already exist. if (clusterSummary == null) { clusterId = emrDao.createEmrCluster(clusterName, emrClusterDefinition, awsParamsDto); emrClusterCreated = true; EmrClusterCreationLogEntity emrClusterCreationLogEntity = new EmrClusterCreationLogEntity(); emrClusterCreationLogEntity.setNamespace(emrClusterDefinitionEntity.getNamespace()); emrClusterCreationLogEntity.setEmrClusterDefinitionName(emrClusterDefinitionEntity.getName()); emrClusterCreationLogEntity.setEmrClusterName(emrClusterAlternateKeyDto.getEmrClusterName()); emrClusterCreationLogEntity.setEmrClusterId(clusterId); emrClusterCreationLogEntity .setEmrClusterDefinition(xmlHelper.objectToXml(emrClusterDefinition)); dmDao.saveAndRefresh(emrClusterCreationLogEntity); } // If the cluster already exist. else { clusterId = clusterSummary.getId(); emrClusterCreated = false; emrClusterDefinition = null; // Do not include definition in response } emrClusterStatus = emrDao.getEmrClusterStatusById(clusterId, awsParamsDto); } catch (AmazonServiceException ex) { handleAmazonException(ex, "An Amazon exception occurred while creating EMR cluster with name \"" + clusterName + "\"."); } } // If the dryRun flag is true and not null else { emrClusterCreated = false; } return createEmrClusterFromRequest(clusterId, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), emrClusterStatus, emrClusterCreated, request.isDryRun(), emrClusterDefinition); }
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * Runs oozie job on an existing EMR Cluster. * * @param request the Run oozie workflow request * * @return the oozie workflow job that was submitted. * @throws Exception if there were any errors while submitting the job. *///from www. jav a 2 s . c o m protected OozieWorkflowJob runOozieWorkflowImpl(RunOozieWorkflowRequest request) throws Exception { // Perform the request validation. validateRunOozieWorkflowRequest(request); String namespace = request.getNamespace(); String emrClusterDefinitionName = request.getEmrClusterDefinitionName(); String emrClusterName = request.getEmrClusterName(); ClusterSummary clusterSummary = getRunningOrWaitingEmrCluster(namespace, emrClusterDefinitionName, emrClusterName); String emrClusterPrivateIpAddress = getEmrClusterMasterIpAddress(clusterSummary.getId()); String jobId = oozieDao.runOozieWorkflow(emrClusterPrivateIpAddress, request.getWorkflowLocation(), request.getParameters()); OozieWorkflowJob oozieWorkflowJob = new OozieWorkflowJob(); oozieWorkflowJob.setId(jobId); oozieWorkflowJob.setNamespace(namespace); oozieWorkflowJob.setEmrClusterDefinitionName(emrClusterDefinitionName); oozieWorkflowJob.setEmrClusterName(emrClusterName); return oozieWorkflowJob; }
From source file:org.finra.dm.service.impl.EmrServiceImpl.java
License:Apache License
/** * Get the oozie workflow.//from w w w . j a v a2 s. c o m * * @param namespace the namespace * @param emrClusterDefinitionName the EMR cluster definition name * @param emrClusterName the EMR cluster name * @param oozieWorkflowJobId the ooxie workflow Id. * @param verbose the flag to indicate whether to return verbose information * * @return OozieWorkflowJob OozieWorkflowJob * @throws Exception Exception */ protected OozieWorkflowJob getEmrOozieWorkflowJobImpl(String namespace, String emrClusterDefinitionName, String emrClusterName, String oozieWorkflowJobId, Boolean verbose) throws Exception { // Validate parameters Assert.isTrue(StringUtils.hasText(namespace), "Namespace is required"); Assert.isTrue(StringUtils.hasText(emrClusterDefinitionName), "EMR cluster definition name is required"); Assert.isTrue(StringUtils.hasText(emrClusterName), "EMR cluster name is required"); Assert.isTrue(StringUtils.hasText(oozieWorkflowJobId), "Oozie workflow job ID is required"); // Trim string parameters String namespaceTrimmed = namespace.trim(); String emrClusterDefinitionNameTrimmed = emrClusterDefinitionName.trim(); String emrClusterNameTrimmed = emrClusterName.trim(); String oozieWorkflowJobIdTrimmed = oozieWorkflowJobId.trim(); // Retrieve cluster's master instance IP ClusterSummary emrClusterSummary = getRunningOrWaitingEmrCluster(namespaceTrimmed, emrClusterDefinitionNameTrimmed, emrClusterNameTrimmed); String masterIpAddress = getEmrClusterMasterIpAddress(emrClusterSummary.getId()); // Retrieve the wrapper oozie workflow. This workflow is the workflow that DM wraps the client's workflow to help copy client workflow definition from // S3 to HDFS. WorkflowJob wrapperWorkflowJob = oozieDao.getEmrOozieWorkflow(masterIpAddress, oozieWorkflowJobIdTrimmed); // Check to make sure that the workflow job is a DM wrapper workflow. Assert.isTrue(wrapperWorkflowJob.getAppName().equals(OozieDaoImpl.DM_OOZIE_WRAPPER_WORKFLOW_NAME), "The oozie workflow with job ID '" + oozieWorkflowJobIdTrimmed + "' is not created by DM. Please ensure that the workflow was created through DM."); // Retrieve the client workflow's job action by navigating through the actions of the wrapper workflow. The client's workflow job ID is represented as // the action's external ID. WorkflowAction clientWorkflowAction = emrHelper.getClientWorkflowAction(wrapperWorkflowJob); WorkflowJob clientWorkflowJob = null; String clientWorkflowStatus = null; boolean hasClientWorkflowInfo = false; WorkflowAction errorWrapperWorkflowAction = null; /* * If the client workflow action is not found, there are three possibilities: * 1. DM_PREP: The client workflow has not yet been run. * 2. DM_FAILED : DM wrapper workflow failed to run successfully. * 3. If client workflow action is found but does not have the external ID, means that it failed to kick off the client workflow. Possible causes: * 3.1 workflow.xml is not present in the location provided. * 3.2 workflow.xml is not a valid workflow. */ if (clientWorkflowAction == null || clientWorkflowAction.getExternalId() == null) { // If wrapper workflow is FAILED/KILLED, means wrapper failed without running client workflow // else DM_PREP if (wrapperWorkflowJob.getStatus().equals(WorkflowJob.Status.KILLED) || wrapperWorkflowJob.getStatus().equals(WorkflowJob.Status.FAILED)) { clientWorkflowStatus = OozieDaoImpl.OOZIE_WORKFLOW_JOB_STATUS_DM_FAILED; // Get error information. errorWrapperWorkflowAction = emrHelper.getFirstWorkflowActionInError(wrapperWorkflowJob); } else { clientWorkflowStatus = OozieDaoImpl.OOZIE_WORKFLOW_JOB_STATUS_DM_PREP; } } else { // Retrieve the client workflow clientWorkflowJob = oozieDao.getEmrOozieWorkflow(masterIpAddress, clientWorkflowAction.getExternalId()); hasClientWorkflowInfo = true; } // Construct result OozieWorkflowJob resultOozieWorkflowJob = new OozieWorkflowJob(); resultOozieWorkflowJob.setId(oozieWorkflowJobId); resultOozieWorkflowJob.setNamespace(namespace); resultOozieWorkflowJob.setEmrClusterDefinitionName(emrClusterDefinitionName); resultOozieWorkflowJob.setEmrClusterName(emrClusterName); // If client workflow is information is not available that DM wrapper workflow has not started the client workflow yet or failed to start. // Hence return status that it is still in DM preparation. if (!hasClientWorkflowInfo) { resultOozieWorkflowJob.setStatus(clientWorkflowStatus); if (errorWrapperWorkflowAction != null) { resultOozieWorkflowJob.setErrorCode(errorWrapperWorkflowAction.getErrorCode()); resultOozieWorkflowJob.setErrorMessage(errorWrapperWorkflowAction.getErrorMessage()); } } else { resultOozieWorkflowJob.setStartTime(toXmlGregorianCalendar(clientWorkflowJob.getStartTime())); resultOozieWorkflowJob.setEndTime(toXmlGregorianCalendar(clientWorkflowJob.getEndTime())); resultOozieWorkflowJob.setStatus(clientWorkflowJob.getStatus().toString()); // Construct actions in the result if verbose flag is explicitly true, default to false if (Boolean.TRUE.equals(verbose)) { List<OozieWorkflowAction> oozieWorkflowActions = new ArrayList<>(); for (WorkflowAction workflowAction : clientWorkflowJob.getActions()) { OozieWorkflowAction resultOozieWorkflowAction = new OozieWorkflowAction(); resultOozieWorkflowAction.setId(workflowAction.getId()); resultOozieWorkflowAction.setName(workflowAction.getName()); resultOozieWorkflowAction.setStartTime(toXmlGregorianCalendar(workflowAction.getStartTime())); resultOozieWorkflowAction.setEndTime(toXmlGregorianCalendar(workflowAction.getEndTime())); resultOozieWorkflowAction.setStatus(workflowAction.getStatus().toString()); resultOozieWorkflowAction.setErrorCode(workflowAction.getErrorCode()); resultOozieWorkflowAction.setErrorMessage(workflowAction.getErrorMessage()); oozieWorkflowActions.add(resultOozieWorkflowAction); } resultOozieWorkflowJob.setWorkflowActions(oozieWorkflowActions); } } return resultOozieWorkflowJob; }
From source file:org.finra.herd.dao.helper.EmrHelper.java
License:Apache License
/** * Gets the ID of an active EMR cluster which matches the given criteria. If both cluster ID and cluster name is specified, the name of the actual cluster * with the given ID must match the specified name. For cases where the cluster is not found (does not exists or not active), the method fails. All * parameters are case-insensitive and whitespace trimmed. Blank parameters are equal to null. * * @param emrClusterId EMR cluster ID/* w w w .ja v a2s .c o m*/ * @param emrClusterName EMR cluster name * @param accountId the account Id that EMR cluster is running under * * @return The cluster ID */ public String getActiveEmrClusterId(String emrClusterId, String emrClusterName, String accountId) { boolean emrClusterIdSpecified = StringUtils.isNotBlank(emrClusterId); boolean emrClusterNameSpecified = StringUtils.isNotBlank(emrClusterName); Assert.isTrue(emrClusterIdSpecified || emrClusterNameSpecified, "One of EMR cluster ID or EMR cluster name must be specified."); AwsParamsDto awsParamsDto = getAwsParamsDtoByAccountId(accountId); // Get cluster by ID first if (emrClusterIdSpecified) { String emrClusterIdTrimmed = emrClusterId.trim(); // Assert cluster exists Cluster cluster = emrDao.getEmrClusterById(emrClusterIdTrimmed, awsParamsDto); Assert.notNull(cluster, String.format("The cluster with ID \"%s\" does not exist.", emrClusterIdTrimmed)); // Assert the cluster's state is active String emrClusterState = cluster.getStatus().getState(); Assert.isTrue(isActiveEmrState(emrClusterState), String.format( "The cluster with ID \"%s\" is not active. The cluster state must be in one of %s. Current state is \"%s\"", emrClusterIdTrimmed, Arrays.toString(getActiveEmrClusterStates()), emrClusterState)); // Assert cluster name equals if cluster name was specified if (emrClusterNameSpecified) { String emrClusterNameTrimmed = emrClusterName.trim(); Assert.isTrue(cluster.getName().equalsIgnoreCase(emrClusterNameTrimmed), String.format( "The cluster with ID \"%s\" does not match the expected name \"%s\". The actual name is \"%s\".", cluster.getId(), emrClusterNameTrimmed, cluster.getName())); } return cluster.getId(); } else { String emrClusterNameTrimmed = emrClusterName.trim(); ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(emrClusterNameTrimmed, awsParamsDto); Assert.notNull(clusterSummary, String.format("The cluster with name \"%s\" does not exist.", emrClusterNameTrimmed)); return clusterSummary.getId(); } }
From source file:org.finra.herd.service.impl.EmrHelperServiceImpl.java
License:Apache License
/** * The implementation of the create cluster AWS specific steps. These steps are run outside of any transaction. * * @param request the EMR cluster create request * @param emrClusterDefinition the EMR cluster definition object * @param emrClusterAlternateKeyDto the EMR cluster alternate key data transfer object * * @return the EMR cluster create data transfer object *//*from ww w. j av a2s . co m*/ EmrClusterCreateDto emrCreateClusterAwsSpecificStepsImpl(EmrClusterCreateRequest request, EmrClusterDefinition emrClusterDefinition, EmrClusterAlternateKeyDto emrClusterAlternateKeyDto) { AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId()); // If instance group definitions are specified, find best price and update definition. if (!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())) { emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); } // The cluster ID record. String clusterId = null; // Is EMR cluster already existing. Boolean emrClusterAlreadyExists = null; // Is EMR cluster created. Boolean emrClusterCreated = null; // EMR cluster status string. String emrClusterStatus = null; // If the dryRun flag is null or false. This is the default option if no flag is given. if (!Boolean.TRUE.equals(request.isDryRun())) { /* * Create the cluster only if the cluster does not already exist. * If the cluster is created, record the newly created cluster ID. * If the cluster already exists, record the existing cluster ID. * If there is any error while attempting to check for existing cluster or create a new one, handle the exception to throw appropriate exception. */ String clusterName = emrHelper.buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { // Try to get an active EMR cluster by its name. ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto); // If cluster does not already exist. if (clusterSummary == null) { clusterId = emrDao.createEmrCluster(clusterName, emrClusterDefinition, awsParamsDto); emrClusterCreated = true; } // If the cluster already exists. else { clusterId = clusterSummary.getId(); emrClusterCreated = false; emrClusterAlreadyExists = true; } emrClusterStatus = emrDao.getEmrClusterStatusById(clusterId, awsParamsDto); } catch (AmazonServiceException ex) { awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while creating EMR cluster with name \"" + clusterName + "\"."); } } // If the dryRun flag is true and not null else { emrClusterCreated = false; } return new EmrClusterCreateDto(clusterId, emrClusterAlreadyExists, emrClusterCreated, emrClusterStatus); }
From source file:org.finra.herd.service.impl.EmrServiceImpl.java
License:Apache License
/** * Gets details of an existing EMR Cluster. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key * @param emrClusterId the cluster id of the cluster to get details * @param emrStepId the step id of the step to get details * @param verbose parameter for whether to return detailed information * @param accountId the optional AWS account that EMR cluster is running in * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets * * @return the EMR Cluster object with details. *///from w ww .ja v a 2 s.c o m protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, String accountId, Boolean retrieveInstanceFleets) { AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAccountId(accountId); // Perform the request validation. validateEmrClusterKey(emrClusterAlternateKeyDto); // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper .getEmrClusterDefinitionEntity(new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName())); EmrCluster emrCluster = createEmrClusterFromRequest(null, emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null); String clusterName = emrHelper.buildEmrClusterName(emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName()); try { // Get Cluster status if clusterId is specified if (StringUtils.isNotBlank(emrClusterId)) { Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto); // Validate that, Cluster exists Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\"."); // Validate that, Cluster name match as specified Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified."); emrCluster.setId(cluster.getId()); setEmrClusterStatus(emrCluster, cluster.getStatus()); } else { ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto); // Validate that, Cluster exists with the name Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\"."); emrCluster.setId(clusterSummary.getId()); setEmrClusterStatus(emrCluster, clusterSummary.getStatus()); } // Get active step details if (emrHelper.isActiveEmrState(emrCluster.getStatus())) { StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto); if (stepSummary != null) { EmrStep activeStep; // If verbose get active step details if (verbose) { activeStep = buildEmrStepFromAwsStep(stepSummary, true); } else { activeStep = buildEmrStepFromAwsStepSummary(stepSummary); } emrCluster.setActiveStep(activeStep); } } // Get requested step details if (StringUtils.isNotBlank(emrStepId)) { Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto); emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose)); } // Get instance fleet if true if (BooleanUtils.isTrue(retrieveInstanceFleets)) { ListInstanceFleetsResult listInstanceFleetsResult = emrDao .getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto); emrCluster.setInstanceFleets( emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult)); } } catch (AmazonServiceException ex) { awsServiceHelper.handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\"."); } return emrCluster; }
From source file:rollsPOC2.util.AWSHelper.java
public static String createOrFindEMRHiveCluster(String clusterName, boolean createWithKeepAlive) throws Exception { String clusterId = null;// w ww . j av a 2s . com AmazonElasticMapReduce emr = AppServices.getEMRClient(); ClusterSummary clusterSummary = findCluster("Treebeard", emr); if (clusterSummary != null) { clusterId = clusterSummary.getId(); System.err.printf("Cluster found with id %s, status %s\n", clusterId, clusterSummary.getStatus().getState()); } if (clusterSummary != null && clusterSummary.getStatus().getState().startsWith("TERMINAT")) { while (findCluster("Treebeard", emr).getStatus().getState().equals("TERMINATING")) { System.out.println("Waiting for previous cluster to terminate"); Thread.sleep(10000l); } System.out.println("Starting cluster..."); StepFactory stepFactory = new StepFactory(); StepConfig enabledebugging = new StepConfig().withName("Enable debugging") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(stepFactory.newEnableDebuggingStep()); // Possibly redundant with ".withApplications(new Application().withName("Hive"))" // StepConfig installHive = new StepConfig() // .withName("Install Hive") // .withActionOnFailure("TERMINATE_JOB_FLOW") // .withHadoopJarStep(stepFactory.newInstallHiveStep()); RunJobFlowRequest request = new RunJobFlowRequest().withName("Treebeard").withReleaseLabel("emr-4.6.0") .withApplications(new Application().withName("Hive")).withSteps(enabledebugging) .withVisibleToAllUsers(true) .withLogUri("s3://aws-logs-800327301943-us-east-1/elasticmapreduce/") .withServiceRole("EMR_DefaultRole").withJobFlowRole("EMR_EC2_DefaultRole") .withInstances(new JobFlowInstancesConfig().withEc2KeyName("bjss").withInstanceCount(2) .withMasterInstanceType("m3.xlarge").withSlaveInstanceType("m1.large") .withKeepJobFlowAliveWhenNoSteps(createWithKeepAlive)); RunJobFlowResult createClusterResult = emr.runJobFlow(request); clusterId = createClusterResult.getJobFlowId(); System.out.printf("Started cluster with id %s\n", clusterId); } return clusterId; }