List of usage examples for com.amazonaws.services.elasticmapreduce.model JobFlowDetail getInstances
public JobFlowInstancesDetail getInstances()
Describes the Amazon EC2 instances of the job flow.
From source file:com.tstordyallison.ffmpegmr.emr.JobflowConfiguration.java
License:Open Source License
public JobflowConfiguration() { try {/*w ww . java 2 s. c o m*/ Gson gson = new GsonBuilder().disableHtmlEscaping().create(); jobflow = gson.fromJson(new BufferedReader(new FileReader(JOB_FLOW_INFO_FILE)), JobflowConfiguration.JobFlow.class); instance = gson.fromJson(new BufferedReader(new FileReader(INSTANCE_INFO_FILE)), JobflowConfiguration.Instance.class); // We must also go and get an up to date value for the coreInstanceCount using the API. DescribeJobFlowsResult jobFlowDescription = JobController.getEmr() .describeJobFlows(new DescribeJobFlowsRequest().withJobFlowIds(jobflow.jobFlowId)); JobFlowDetail jobFlowDetail = jobFlowDescription.getJobFlows().get(0); Collection<InstanceGroupDetail> instanceGroups = jobFlowDetail.getInstances().getInstanceGroups(); for (InstanceGroupDetail instanceGroup : instanceGroups) { if (instanceGroup.getInstanceRole().equals("CORE")) { jobflow.coreInstanceCount = instanceGroup.getInstanceRunningCount(); break; } } } catch (Exception e) { System.err.println("WARNING: Unable to read instance configuration files - logs will be inaccurate."); } }
From source file:datameer.awstasks.aws.emr.EmrCluster.java
License:Apache License
private void waitUntilClusterStarted(final String jobFlowId) throws InterruptedException { doWhileNot(new Callable<Boolean>() { @Override/*from w ww . j av a 2 s .com*/ public Boolean call() throws Exception { JobFlowDetail jobFlowDetail = getJobFlowDetail(jobFlowId); JobFlowState state = JobFlowState.valueOf(jobFlowDetail.getExecutionStatusDetail().getState()); LOG.info("elastic cluster '" + jobFlowDetail.getName() + "/" + jobFlowId + "' in state '" + state + "'"); boolean finished = state != JobFlowState.STARTING && state != JobFlowState.BOOTSTRAPPING; if (finished) { _masterHost = jobFlowDetail.getInstances().getMasterPublicDnsName(); _instanceCount = jobFlowDetail.getInstances().getInstanceCount(); if (!state.isOperational()) { throw new IllegalStateException( "starting of job flow '" + jobFlowId + "' failed with state '" + state + "'"); } _startTime = jobFlowDetail.getExecutionStatusDetail().getStartDateTime().getTime(); } return finished; } }, getRequestInterval()); }
From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java
License:Apache License
/** * @param config/*from w w w . j a v a 2 s. c o m*/ * @throws URISyntaxException */ public void runJob(Config config) throws URISyntaxException { RunJobFlowRequest runJobFlowRequest = null; CreateStepConfigger csc = getCreateStepConfigger(config); if (csc == null) { log.error("Step config create error"); return; } if (jobFlowId == null) { runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME) .withBootstrapActions( new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)), new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI) .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")), new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path()))) .withInstances(setupJobFlowInstancesConfig()); if (!isEmpty(emrProperties.getLogUri())) { runJobFlowRequest.setLogUri(emrProperties.getLogUri()); } List<StepConfig> stepConfigs = new ArrayList<StepConfig>(); if (emrProperties.isDebug()) { StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME) .withActionOnFailure(ACTION_ON_TERMINATE) .withHadoopJarStep(new StepFactory().newEnableDebuggingStep()); stepConfigs.add(enableDebugging); } for (StepConfig sc : csc.createStepConfig(config)) { stepConfigs.add(sc); } runJobFlowRequest.setSteps(stepConfigs); try { RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest); jobFlowId = result.getJobFlowId(); checkDate = new Date(); } catch (Exception e) { e.printStackTrace(); log.error(e); } } else { AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId) .withSteps(csc.createStepConfig(config)); emr.addJobFlowSteps(addJobFlowStepsRequest); } running = true; try { config.setJobFlowId(jobFlowId); QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } int stepSize = 0; String stepStatus = JobUtils.STEP_STATUS_PENDING; while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) { if (sleep()) { break; } DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest() .withJobFlowIds(jobFlowId); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); if (describeJobFlowsResult.getJobFlows().size() != 1) { break; } JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0); JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances(); masterPublicDnsName = instancesDetail.getMasterPublicDnsName(); if (isEmpty(config.getMasterPublicDnsName())) { try { config.setMasterPublicDnsName(masterPublicDnsName); QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } } stepSize = jobFlowDetail.getSteps().size(); for (StepDetail stepDetail : jobFlowDetail.getSteps()) { if (stepDetail.getStepConfig().getName().equals(config.getName())) { stepStatus = stepDetail.getExecutionStatusDetail().getState(); break; } } } if (config.isDeleteOnExit()) { if (config.getJobType() == Config.JOB_TYPE_STREAMING) { S3Utils.delete(s3, config.getArgMap().get("mapper")); S3Utils.delete(s3, config.getArgMap().get("reducer")); } else { S3Utils.delete(s3, config.getRun()); } } // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV) if (stepSize >= 255) { instanceTerminate(); } running = false; if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) { config.setStatus(Config.JOB_STATUS_COMPLETE); } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) { config.setStatus(Config.JOB_STATUS_ERROR); } else if (terminated) { config.setStatus(Config.JOB_STATUS_CANCEL); } try { QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } }
From source file:org.huahinframework.emanager.util.JobUtils.java
License:Apache License
public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) { DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); if (describeJobFlowsResult.getJobFlows().size() != 1) { return new JSONObject(); }//from w w w . j a v a 2 s . c o m JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0); JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail(); JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances(); Map<String, Object> m = new HashMap<String, Object>(); m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId()); m.put(Response.STATE, executionStatusDetail.getState()); m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString()); m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true)); m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true)); m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false)); m.put(Response.NAME, jobFlowDetail.getName()); m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false)); if (!isEmpty(jobFlowDetail.getSupportedProducts())) { m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts()); } m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false)); m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false)); m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false)); m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount())); m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS, object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true)); m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false)); m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false)); m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false)); m.put(Response.AVAILABILITY_ZONE, object2String(instancesDetail.getPlacement().getAvailabilityZone(), false)); m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false)); if (!isEmpty(jobFlowDetail.getBootstrapActions())) { List<Object> l = new ArrayList<Object>(); for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) { Map<String, Object> mm = new HashMap<String, Object>(); BootstrapActionConfig bac = bad.getBootstrapActionConfig(); ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction(); mm.put(Response.NAME, object2String(bac.getName(), false)); mm.put(Response.PATH, object2String(sbac.getPath(), false)); if (!isEmpty(sbac.getArgs())) { mm.put(Response.ARGS, sbac.getArgs()); } l.add(mm); } m.put(Response.BOOTSTRAP_ACTIONS, l); } if (!isEmpty(jobFlowDetail.getSteps())) { List<Object> l = new ArrayList<Object>(); for (StepDetail sd : jobFlowDetail.getSteps()) { Map<String, Object> mm = new HashMap<String, Object>(); StepConfig sc = sd.getStepConfig(); StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail(); mm.put(Response.NAME, sc.getName()); mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure()); mm.put(Response.STATE, object2String(sesd.getState(), false)); mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true)); mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true)); mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true)); HadoopJarStepConfig hjsc = sc.getHadoopJarStep(); mm.put(Response.JAR, object2String(hjsc.getJar(), false)); mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false)); if (!isEmpty(hjsc.getArgs())) { mm.put(Response.ARGS, hjsc.getArgs()); } l.add(mm); } m.put(Response.STEPS, l); } return new JSONObject(m); }