List of usage examples for org.apache.hadoop.mapreduce Cluster getAllJobStatuses
public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException
From source file:com.cloudera.oryx.computation.common.DistributedGenerationRunner.java
License:Open Source License
private static Collection<String> find(String instanceDir) throws IOException, InterruptedException { Collection<String> result = Lists.newArrayList(); // This is where we will see Hadoop config problems first, so log extra info Cluster cluster; try {//from www .j a v a2 s . c o m cluster = new Cluster(OryxConfiguration.get()); } catch (IOException ioe) { log.error("Unable to init the Hadoop cluster. Check that an MR2, not MR1, cluster is configured."); throw ioe; } try { JobStatus[] statuses = cluster.getAllJobStatuses(); if (statuses != null) { for (JobStatus jobStatus : statuses) { JobStatus.State state = jobStatus.getState(); if (state == JobStatus.State.RUNNING || state == JobStatus.State.PREP) { Job job = cluster.getJob(jobStatus.getJobID()); if (job != null) { String jobName = job.getJobName(); log.info("Found running job {}", jobName); if (jobName.startsWith("Oryx-" + instanceDir + '-')) { result.add(jobName); } } } } } } finally { cluster.close(); } return result; }
From source file:com.ikanow.aleph2.analytics.hadoop.services.HadoopTechnologyService.java
License:Apache License
@Override public CompletableFuture<BasicMessageBean> stopAnalyticJob(DataBucketBean analytic_bucket, Collection<AnalyticThreadJobBean> jobs, AnalyticThreadJobBean job_to_stop, IAnalyticsContext context) { try {// ww w . j av a 2 s .c o m final Cluster cluster = new Cluster(_config.get()); final String job_name = BucketUtils.getUniqueSignature(analytic_bucket.full_name(), Optional.ofNullable(job_to_stop.name())); return Arrays.stream(cluster.getAllJobStatuses()) .filter(job_status -> job_status.getJobName().equals(job_name)).findFirst() .map(Lambdas.wrap_u(job_status -> { final Job job = cluster.getJob(job_status.getJobID()); job.killJob(); return CompletableFuture .completedFuture(ErrorUtils.buildSuccessMessage(this.getClass().getSimpleName(), "stopAnalyticJob", analytic_bucket.full_name() + ":" + job_to_stop.name())); })).get() // (Will throw if not found falling through to catch below) ; } catch (Throwable t) { return CompletableFuture.completedFuture( ErrorUtils.buildSuccessMessage(this.getClass().getSimpleName(), "stopAnalyticJob", HadoopErrorUtils.JOB_STOP_ERROR, job_to_stop.name(), analytic_bucket.full_name())); } }
From source file:com.ikanow.aleph2.analytics.hadoop.services.HadoopTechnologyService.java
License:Apache License
@Override public ManagementFuture<Boolean> checkAnalyticJobProgress(DataBucketBean analytic_bucket, Collection<AnalyticThreadJobBean> jobs, AnalyticThreadJobBean job_to_check, IAnalyticsContext context) { try {//from w w w . ja v a2 s.c o m final Cluster cluster = new Cluster(_config.get()); final String job_name = BucketUtils.getUniqueSignature(analytic_bucket.full_name(), Optional.ofNullable(job_to_check.name())); return Arrays.stream(cluster.getAllJobStatuses()) .filter(job_status -> job_status.getJobName().equals(job_name)).findFirst().map(job_status -> { //TODO (ALEPH-12): create useful info in the side channel beans ... eg if it's an error? // (need to get the job first, then get more info) return FutureUtils.createManagementFuture( CompletableFuture.completedFuture(job_status.isJobComplete()), CompletableFuture.completedFuture( Arrays.asList(ErrorUtils.buildMessage(true, this.getClass().getSimpleName(), "checkAnalyticJobProgress", "TBD: more status")))); }).get() // (Will throw if not found falling through to catch below) ; } catch (Throwable t) { return FutureUtils.createManagementFuture(CompletableFuture.completedFuture(true)); } }
From source file:org.huahinframework.manager.rest.service.JobService.java
License:Apache License
/** * @return {@link JSONObject}/*from w ww . jav a 2s . co m*/ */ @Path("/kill/id/{" + JOBID + "}") @DELETE @Produces(MediaType.APPLICATION_JSON) public JSONObject killJobId(@PathParam(JOBID) String jobId) { Map<String, String> status = new HashMap<String, String>(); try { Cluster cluster = new Cluster(getJobConf()); for (JobStatus jobStatus : cluster.getAllJobStatuses()) { if (jobStatus.getJobID().toString().equals(jobId)) { Job job = cluster.getJob(jobStatus.getJobID()); if (job == null) { break; } job.killJob(); status.put(Response.STATUS, "Killed job " + jobId); break; } } } catch (Exception e) { e.printStackTrace(); log.error(e); status.put(Response.STATUS, e.getMessage()); } if (status.isEmpty()) { status.put(Response.STATUS, "Could not find job " + jobId); } return new JSONObject(status); }
From source file:org.huahinframework.manager.rest.service.JobService.java
License:Apache License
/** * @return {@link JSONObject}/*from w ww . j a va 2 s.c o m*/ */ @Path("/kill/name/{" + JOBNAME + "}") @DELETE @Produces(MediaType.APPLICATION_JSON) public JSONObject killJobName(@PathParam(JOBNAME) String jobName) { Map<String, String> status = new HashMap<String, String>(); try { Cluster cluster = new Cluster(getJobConf()); for (JobStatus jobStatus : cluster.getAllJobStatuses()) { Job job = cluster.getJob(jobStatus.getJobID()); if (job == null) { break; } if (job.getJobName().equals(jobName)) { job.killJob(); status.put(Response.STATUS, "Killed job " + jobName); break; } } } catch (Exception e) { e.printStackTrace(); log.error(e); status.put(Response.STATUS, e.getMessage()); } if (status.isEmpty()) { status.put(Response.STATUS, "Could not find job " + jobName); } return new JSONObject(status); }
From source file:org.huahinframework.manager.rest.service.JobService.java
License:Apache License
/** * @param jobId//www . ja v a2s . co m * @return {@link JSONObject} * @throws IOException * @throws InterruptedException */ private Map<String, Object> getStatus(String jobId) throws IOException, InterruptedException { Map<String, Object> job = null; Cluster cluster = new Cluster(getJobConf()); for (JobStatus jobStatus : cluster.getAllJobStatuses()) { if (jobStatus.getJobID().toString().equals(jobId)) { job = JobUtils.getJob(jobStatus); Job j = cluster.getJob(jobStatus.getJobID()); if (j == null) { break; } Calendar finishTime = Calendar.getInstance(); finishTime.setTimeInMillis(j.getFinishTime()); job.put(Response.FINISH_TIME, finishTime.getTime().toString()); Map<String, Map<String, Long>> groups = new HashMap<String, Map<String, Long>>(); for (String s : j.getCounters().getGroupNames()) { CounterGroup counterGroup = j.getCounters().getGroup(s); Iterator<Counter> ite = counterGroup.iterator(); Map<String, Long> counters = new HashMap<String, Long>(); groups.put(counterGroup.getDisplayName(), counters); while (ite.hasNext()) { Counter counter = (Counter) ite.next(); counters.put(counter.getDisplayName(), counter.getValue()); } } job.put(Response.GROUPS, groups); break; } } return job; }
From source file:org.huahinframework.manager.util.JobUtils.java
License:Apache License
/** * @param state/* ww w.ja v a 2 s .c o m*/ * @param conf * @return {@link List} of {@link JSONObject} * @throws IOException * @throws InterruptedException */ public static List<JSONObject> listJob(State state, JobConf conf) throws IOException, InterruptedException { List<JSONObject> l = new ArrayList<JSONObject>(); Cluster cluster = new Cluster(conf); for (JobStatus jobStatus : cluster.getAllJobStatuses()) { jobStatus.getState(); if (state == null || state == jobStatus.getState()) { Map<String, Object> m = getJob(jobStatus); if (m != null) { l.add(new JSONObject(m)); } } } return l; }