List of usage examples for org.apache.hadoop.mapreduce JobID forName
public static JobID forName(String str) throws IllegalArgumentException
From source file:com.google.appengine.tools.mapreduce.AppEngineJobContext.java
License:Apache License
/** * Gets the Job ID from the given request. * * @param req a servlet request with the job ID stored in the * {@link #JOB_ID_PARAMETER_NAME} parameter * @return the job ID// w w w .j ava 2 s. c om */ // VisibleForTesting static JobID getJobIDFromRequest(HttpServletRequest req) { String jobIdString = req.getParameter(JOB_ID_PARAMETER_NAME); if (jobIdString == null) { throw new RuntimeException("Couldn't get Job ID for request. Aborting!"); } return JobID.forName(jobIdString); }
From source file:com.google.appengine.tools.mapreduce.MapReduceState.java
License:Apache License
/** * Marks the job as ABORTED and removes all ShardStates *//*from www . ja v a2 s . co m*/ public void abort() { entity.setProperty(STATUS_PROPERTY, Status.ABORTED.name()); persist(); List<ShardState> shards = ShardState.getShardStatesFromJobID(service, JobID.forName(getJobID())); List<Key> shardKeys = Lists.newArrayList(); for (ShardState shard : shards) { shardKeys.add(shard.getKey()); } service.delete(shardKeys); }
From source file:com.google.appengine.tools.mapreduce.MapReduceState.java
License:Apache License
/** * Create json object from this one. If detailed is true creates an object * with all the information needed for the job detail status view. Otherwise, * only includes the overview information. *///from ww w . j a v a 2s . c o m public JSONObject toJson(boolean detailed) { JSONObject jobObject = new JSONObject(); try { jobObject.put("name", getName()); jobObject.put("mapreduce_id", getJobID().toString()); jobObject.put("active", getStatus() == MapReduceState.Status.ACTIVE); jobObject.put("updated_timestamp_ms", getLastPollTime()); jobObject.put("start_timestamp_ms", getStartTime()); jobObject.put("result_status", "" + getStatus()); if (detailed) { jobObject.put("counters", toJson(getCounters())); jobObject.put("configuration", getConfigurationXML()); jobObject.put("chart_url", getChartUrl()); // TODO(frew): Fill this from the Configuration JSONObject mapperSpec = new JSONObject(); mapperSpec.put("mapper_params", new JSONObject()); jobObject.put("mapper_spec", mapperSpec); List<ShardState> shardStates = ShardState.getShardStatesFromJobID(service, JobID.forName(getJobID())); JSONArray shardArray = new JSONArray(); for (ShardState shardState : shardStates) { shardArray.put(shardState.toJson()); } jobObject.put("shards", shardArray); } else { jobObject.put("shards", getShardCount()); jobObject.put("active_shards", getActiveShardCount()); } } catch (JSONException e) { throw new RuntimeException("Hard coded string is null", e); } return jobObject; }
From source file:com.google.appengine.tools.mapreduce.v2.impl.MapReduceState.java
License:Apache License
/** * Create json object from this one. If detailed is true creates an object * with all the information needed for the job detail status view. Otherwise, * only includes the overview information. *//* ww w . j ava2 s. co m*/ public JSONObject toJson(boolean detailed) { JSONObject jobObject = new JSONObject(); try { jobObject.put("name", getName()); jobObject.put("mapreduce_id", getJobID().toString()); jobObject.put("active", getStatus() == MapReduceState.Status.ACTIVE); jobObject.put("updated_timestamp_ms", getLastPollTime()); jobObject.put("start_timestamp_ms", getStartTime()); jobObject.put("result_status", "" + getStatus()); if (detailed) { jobObject.put("counters", toJson(getCounters())); jobObject.put("configuration", getConfigurationXML()); jobObject.put("chart_url", getChartUrl()); // TODO(user): Fill this from the Configuration JSONObject mapperSpec = new JSONObject(); mapperSpec.put("mapper_params", new JSONObject()); jobObject.put("mapper_spec", mapperSpec); List<ShardState> shardStates = ShardState.getShardStatesFromJobID(service, JobID.forName(getJobID())); JSONArray shardArray = new JSONArray(); for (ShardState shardState : shardStates) { shardArray.put(shardState.toJson()); } jobObject.put("shards", shardArray); } else { jobObject.put("shards", getShardCount()); jobObject.put("active_shards", getActiveShardCount()); } } catch (JSONException e) { throw new RuntimeException("Hard coded string is null", e); } return jobObject; }
From source file:crunch.MaxTemperature.java
License:Apache License
@Override
public int run(String[] args) throws Exception {
if (args.length != 1) {
JobBuilder.printUsage(this, "<job ID>");
return -1;
}/* w w w.j ava2s. co m*/
String jobID = args[0];
JobClient jobClient = new JobClient(new JobConf(getConf()));
RunningJob job = jobClient.getJob(JobID.forName(jobID));
if (job == null) {
System.err.printf("No job with ID %s found.\n", jobID);
return -1;
}
if (!job.isComplete()) {
System.err.printf("Job %s is not complete.\n", jobID);
return -1;
}
Counters counters = job.getCounters();
long missing = counters.getCounter(MaxTemperatureWithCounters.Temperature.MISSING);
long total = counters.getCounter(Task.Counter.MAP_INPUT_RECORDS);
System.out.printf("Records with missing temperature fields: %.2f%%\n", 100.0 * missing / total);
return 0;
}
From source file:crunch.MaxTemperature.java
License:Apache License
@Override
public int run(String[] args) throws Exception {
if (args.length != 1) {
JobBuilder.printUsage(this, "<job ID>");
return -1;
}// w w w. j a v a 2 s.co m
String jobID = args[0];
// vv NewMissingTemperatureFields
Cluster cluster = new Cluster(getConf());
Job job = cluster.getJob(JobID.forName(jobID));
// ^^ NewMissingTemperatureFields
if (job == null) {
System.err.printf("No job with ID %s found.\n", jobID);
return -1;
}
if (!job.isComplete()) {
System.err.printf("Job %s is not complete.\n", jobID);
return -1;
}
// vv NewMissingTemperatureFields
Counters counters = job.getCounters();
long missing = counters.findCounter(MaxTemperatureWithCounters.Temperature.MISSING).getValue();
long total = counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue();
// ^^ NewMissingTemperatureFields
System.out.printf("Records with missing temperature fields: %.2f%%\n", 100.0 * missing / total);
return 0;
}
From source file:crunch.MaxTemperature.java
License:Apache License
@Override
public int run(String[] args) throws Exception {
if (args.length != 1) {
JobBuilder.printUsage(this, "<job ID>");
return -1;
}// w w w.j a v a 2 s. c om
String jobID = args[0];
JobClient jobClient = new JobClient(new JobConf(getConf()));
RunningJob job = jobClient.getJob(JobID.forName(jobID));
if (job == null) {
System.err.printf("No job with ID %s found.\n", jobID);
return -1;
}
if (!job.isComplete()) {
System.err.printf("Job %s is not complete.\n", jobID);
return -1;
}
Counters counters = job.getCounters();
long missing = counters.getCounter(MaxTemperatureWithCounters.Temperature.MISSING);
long total = counters.findCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS")
.getCounter();
System.out.printf("Records with missing temperature fields: %.2f%%\n", 100.0 * missing / total);
return 0;
}
From source file:org.apache.falcon.logging.TaskLogRetrieverYarn.java
License:Apache License
@Override public List<String> retrieveTaskLogURL(String jobIdStr) throws IOException { List<String> taskLogUrls = new ArrayList<String>(); Configuration conf = getConf(); Cluster cluster = getCluster(conf);/* w ww. j a v a 2 s . c o m*/ JobID jobID = JobID.forName(jobIdStr); if (jobID == null) { LOG.warn("External id for workflow action is null"); return null; } if (conf.get(YARN_LOG_SERVER_URL) == null) { LOG.warn("YARN log Server is null"); return null; } try { Job job = cluster.getJob(jobID); if (job != null) { TaskCompletionEvent[] events = job.getTaskCompletionEvents(0); for (TaskCompletionEvent event : events) { LogParams params = cluster.getLogParams(jobID, event.getTaskAttemptId()); String url = (conf.get(YARN_LOG_SERVER_URL).startsWith(SCHEME) ? conf.get(YARN_LOG_SERVER_URL) : SCHEME + conf.get(YARN_LOG_SERVER_URL)) + "/" + event.getTaskTrackerHttp() + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner() + "?start=0"; LOG.info("Task Log URL for the job {} is {}" + jobIdStr, url); taskLogUrls.add(url); } return taskLogUrls; } LOG.warn("Unable to find the job in cluster {}" + jobIdStr); return null; } catch (InterruptedException e) { throw new IOException(e); } }
From source file:org.apache.falcon.logging.v2.TaskLogRetrieverYarn.java
License:Apache License
@Override public List<String> retrieveTaskLogURL(String jobIdStr) throws IOException { List<String> taskLogUrls = new ArrayList<String>(); Configuration conf = getConf(); Cluster cluster = getCluster(conf);/*from w w w. j a v a 2 s. c o m*/ JobID jobID = JobID.forName(jobIdStr); if (jobID == null) { LOG.warn("External id for workflow action is null"); return null; } try { Job job = cluster.getJob(jobID); if (job != null) { TaskCompletionEvent[] events = job.getTaskCompletionEvents(0); for (TaskCompletionEvent event : events) { LogParams params = cluster.getLogParams(jobID, event.getTaskAttemptId()); String url = SCHEME + conf.get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp() + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner() + "?start=0"; LOG.info("Task Log URL for the job {} is {}" + jobIdStr, url); taskLogUrls.add(url); } return taskLogUrls; } LOG.warn("Unable to find the job in cluster {}" + jobIdStr); return null; } catch (InterruptedException e) { throw new IOException(e); } }
From source file:org.apache.falcon.logging.v2.TaskLogRetrieverYarnTest.java
License:Apache License
@DataProvider(name = "testData") public Object[][] testData() throws IOException, InterruptedException { int samples = getRandomValueInRange(10) + 1; Object[][] resultSet = new Object[samples][2]; for (int count = 0; count < samples; count++) { List<String> expectedResult = new ArrayList<String>(); Cluster cluster = getCluster(getConf()); String jobId = new JobID("job", RANDOM.nextInt(1000)).toString(); boolean success = RANDOM.nextBoolean(); JobID jobID = JobID.forName(jobId); int numEvents = getRandomValueInRange(10) + 1; TaskCompletionEvent[] events = getTaskCompletionEvents(numEvents, jobID); Job job = mock(Job.class); when(cluster.getJob(jobID)).thenReturn(job); when(job.getTaskCompletionEvents(0)).thenReturn(events); for (TaskCompletionEvent event : events) { if (success) { LogParams params = getLogParams(); when(cluster.getLogParams(jobID, event.getTaskAttemptId())).thenReturn(params); String url = SCHEME + getConf().get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp() + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner() + "?start=0"; expectedResult.add(url); } else { when(cluster.getJob(jobID)).thenReturn(null); expectedResult = null;//w ww. j a v a 2s. c o m } resultSet[count] = new Object[] { jobId, expectedResult }; } } return resultSet; }