Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tez.mapreduce; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Map; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobPriority; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.Progress; import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil; import com.google.common.base.Joiner; public class DAGJobStatus extends JobStatus { private final String jobFile; private final DAGStatus dagStatus; private final ApplicationReport report; public DAGJobStatus(ApplicationReport report, DAGStatus dagStatus, String jobFile) { super(); this.dagStatus = dagStatus; this.jobFile = jobFile; this.report = report; } @Override protected synchronized void setMapProgress(float p) { throw new UnsupportedOperationException(); } @Override protected synchronized void setCleanupProgress(float p) { throw new UnsupportedOperationException(); } @Override protected synchronized void setSetupProgress(float p) { throw new UnsupportedOperationException(); } @Override protected synchronized void setReduceProgress(float p) { throw new UnsupportedOperationException(); } @Override protected synchronized void setPriority(JobPriority jp) { throw new UnsupportedOperationException(); } @Override protected synchronized void setFinishTime(long finishTime) { throw new UnsupportedOperationException(); } @Override protected synchronized void setHistoryFile(String historyFile) { throw new UnsupportedOperationException(); } @Override protected synchronized void setTrackingUrl(String trackingUrl) { throw new UnsupportedOperationException(); } @Override protected synchronized void setRetired() { throw new UnsupportedOperationException(); } @Override protected synchronized void setState(State state) { throw new UnsupportedOperationException(); } @Override protected synchronized void setStartTime(long startTime) { throw new UnsupportedOperationException(); } @Override protected synchronized void setUsername(String userName) { throw new UnsupportedOperationException(); } @Override protected synchronized void setSchedulingInfo(String schedulingInfo) { throw new UnsupportedOperationException(); } @Override protected synchronized void setJobACLs(Map<JobACL, AccessControlList> acls) { throw new UnsupportedOperationException(); } @Override protected synchronized void setQueue(String queue) { throw new UnsupportedOperationException(); } @Override protected synchronized void setFailureInfo(String failureInfo) { throw new UnsupportedOperationException(); } @Override public synchronized String getQueue() { return report.getQueue(); } @Override public synchronized float getMapProgress() { if (dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return 1.0f; } return 0.0f; } @Override public synchronized float getCleanupProgress() { if (dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { return 1.0f; } return 0.0f; } @Override public synchronized float getSetupProgress() { if (dagStatus.getState() == DAGStatus.State.RUNNING) { return 1.0f; } return 0.0f; } @Override public synchronized float getReduceProgress() { if (dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getFinalReduceVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return 1.0f; } return 0.0f; } @Override public synchronized State getState() { switch (dagStatus.getState()) { case SUBMITTED: case INITING: return State.PREP; case RUNNING: return State.RUNNING; case SUCCEEDED: return State.SUCCEEDED; case KILLED: return State.KILLED; case FAILED: case ERROR: return State.FAILED; default: throw new TezUncheckedException("Unknown value of DAGState.State:" + dagStatus.getState()); } } @Override public synchronized long getStartTime() { return report.getStartTime(); } @Override public JobID getJobID() { return TypeConverter.fromYarn(report.getApplicationId()); } @Override public synchronized String getUsername() { return report.getUser(); } @Override public synchronized String getSchedulingInfo() { return report.getTrackingUrl(); } @Override public synchronized Map<JobACL, AccessControlList> getJobACLs() { // TODO Auto-generated method stub return super.getJobACLs(); } @Override public synchronized JobPriority getPriority() { // TEX-147: return real priority return JobPriority.NORMAL; } @Override public synchronized String getFailureInfo() { return Joiner.on(". ").join(dagStatus.getDiagnostics()); } @Override public synchronized boolean isJobComplete() { return (dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR); } @Override public synchronized void write(DataOutput out) throws IOException { // FIXME } @Override public synchronized void readFields(DataInput in) throws IOException { // FIXME } @Override public String getJobName() { return report.getName(); } @Override public String getJobFile() { return jobFile; } @Override public synchronized String getTrackingUrl() { return report.getTrackingUrl(); } @Override public synchronized long getFinishTime() { return report.getFinishTime(); } @Override public synchronized boolean isRetired() { // FIXME handle retired jobs? return false; } @Override public synchronized String getHistoryFile() { // FIXME handle history in status return null; } @Override public int getNumUsedSlots() { return report.getApplicationResourceUsageReport().getNumUsedContainers(); } @Override public void setNumUsedSlots(int n) { throw new UnsupportedOperationException(); } @Override public int getNumReservedSlots() { return report.getApplicationResourceUsageReport().getNumReservedContainers(); } @Override public void setNumReservedSlots(int n) { throw new UnsupportedOperationException(); } @Override public int getUsedMem() { return report.getApplicationResourceUsageReport().getUsedResources().getMemory(); } @Override public void setUsedMem(int m) { throw new UnsupportedOperationException(); } @Override public int getReservedMem() { return report.getApplicationResourceUsageReport().getReservedResources().getMemory(); } @Override public void setReservedMem(int r) { throw new UnsupportedOperationException(); } @Override public int getNeededMem() { return report.getApplicationResourceUsageReport().getNeededResources().getMemory(); } @Override public void setNeededMem(int n) { throw new UnsupportedOperationException(); } @Override public synchronized boolean isUber() { return false; } @Override public synchronized void setUber(boolean isUber) { throw new UnsupportedOperationException(); } @Override public String toString() { StringBuffer buffer = new StringBuffer(); buffer.append("job-id : " + getJobID()); buffer.append("uber-mode : " + isUber()); buffer.append("map-progress : " + getMapProgress()); buffer.append("reduce-progress : " + getReduceProgress()); buffer.append("cleanup-progress : " + getCleanupProgress()); buffer.append("setup-progress : " + getSetupProgress()); buffer.append("runstate : " + getState()); buffer.append("start-time : " + getStartTime()); buffer.append("user-name : " + getUsername()); buffer.append("priority : " + getPriority()); buffer.append("scheduling-info : " + getSchedulingInfo()); buffer.append("num-used-slots" + getNumUsedSlots()); buffer.append("num-reserved-slots" + getNumReservedSlots()); buffer.append("used-mem" + getUsedMem()); buffer.append("reserved-mem" + getReservedMem()); buffer.append("needed-mem" + getNeededMem()); return buffer.toString(); } private float getProgress(String vertexName) { Progress progress = dagStatus.getVertexProgress().get(vertexName); if (progress == null) { // no such stage. return 0 like MR app currently does. return 0; } float totalTasks = (float) progress.getTotalTaskCount(); if (totalTasks != 0) { return progress.getSucceededTaskCount() / totalTasks; } return 1; } }