Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.pig.tools.pigstats; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.jobcontrol.Job; import org.apache.pig.ExecType; import org.apache.pig.PigException; import org.apache.pig.PigRunner.ReturnCode; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NativeMapReduceOper; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROpPlanVisitor; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.plan.DependencyOrderWalker; import org.apache.pig.impl.plan.VisitorException; import org.apache.pig.newplan.Operator; import org.apache.pig.newplan.OperatorPlan; import org.apache.pig.newplan.PlanVisitor; import org.apache.pig.tools.pigstats.JobStats.JobState; /** * SimplePigStats encapsulates the statistics collected from a running script. * It includes status of the execution, the DAG of its MR jobs, as well as * information about outputs and inputs of the script. */ final class SimplePigStats extends PigStats { private static final Log LOG = LogFactory.getLog(SimplePigStats.class); private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; private PigContext pigContext; private JobClient jobClient; private JobControlCompiler jcc; private JobGraph jobPlan; private Map<Job, MapReduceOper> jobMroMap; private Map<MapReduceOper, JobStats> mroJobMap; // successful jobs so far private Set<Job> jobSeen = new HashSet<Job>(); private Map<String, OutputStats> aliasOuputMap; private long startTime = -1; private long endTime = -1; private String userId; /** * This class builds the job DAG from a MR plan */ private class JobGraphBuilder extends MROpPlanVisitor { public JobGraphBuilder(MROperPlan plan) { super(plan, new DependencyOrderWalker<MapReduceOper, MROperPlan>(plan)); jobPlan = new JobGraph(); mroJobMap = new HashMap<MapReduceOper, JobStats>(); } @Override public void visitMROp(MapReduceOper mr) throws VisitorException { JobStats js = new JobStats(mr.getOperatorKey().toString(), jobPlan); jobPlan.add(js); List<MapReduceOper> preds = getPlan().getPredecessors(mr); if (preds != null) { for (MapReduceOper pred : preds) { JobStats jpred = mroJobMap.get(pred); if (!jobPlan.isConnected(jpred, js)) { jobPlan.connect(jpred, js); } } } mroJobMap.put(mr, js); } } /** * This class prints a JobGraph */ static class JobGraphPrinter extends PlanVisitor { StringBuffer buf; protected JobGraphPrinter(OperatorPlan plan) { super(plan, new org.apache.pig.newplan.DependencyOrderWalker(plan)); buf = new StringBuffer(); } public void visit(JobStats op) throws FrontendException { buf.append(op.getJobId()); List<Operator> succs = plan.getSuccessors(op); if (succs != null) { buf.append("\t->\t"); for (Operator p : succs) { buf.append(((JobStats) p).getJobId()).append(","); } } buf.append("\n"); } @Override public String toString() { buf.append("\n"); return buf.toString(); } } @Override public List<String> getAllErrorMessages() { throw new UnsupportedOperationException(); } @Override public Map<String, List<PigStats>> getAllStats() { throw new UnsupportedOperationException(); } @Override public boolean isEmbedded() { return false; } @Override public boolean isSuccessful() { return (getNumberJobs() == 0 && returnCode == ReturnCode.UNKNOWN || returnCode == ReturnCode.SUCCESS); } @Override public Properties getPigProperties() { if (pigContext == null) return null; return pigContext.getProperties(); } @Override public JobGraph getJobGraph() { return jobPlan; } @Override public List<String> getOutputLocations() { ArrayList<String> locations = new ArrayList<String>(); for (OutputStats output : getOutputStats()) { locations.add(output.getLocation()); } return Collections.unmodifiableList(locations); } @Override public List<String> getOutputNames() { ArrayList<String> names = new ArrayList<String>(); for (OutputStats output : getOutputStats()) { names.add(output.getName()); } return Collections.unmodifiableList(names); } @Override public long getNumberBytes(String location) { if (location == null) return -1; String name = new Path(location).getName(); long count = -1; for (OutputStats output : getOutputStats()) { if (name.equals(output.getName())) { count = output.getBytes(); break; } } return count; } @Override public long getNumberRecords(String location) { if (location == null) return -1; String name = new Path(location).getName(); long count = -1; for (OutputStats output : getOutputStats()) { if (name.equals(output.getName())) { count = output.getNumberRecords(); break; } } return count; } @Override public String getOutputAlias(String location) { if (location == null) return null; String name = new Path(location).getName(); String alias = null; for (OutputStats output : getOutputStats()) { if (name.equals(output.getName())) { alias = output.getAlias(); break; } } return alias; } @Override public long getSMMSpillCount() { Iterator<JobStats> it = jobPlan.iterator(); long ret = 0; while (it.hasNext()) { ret += it.next().getSMMSpillCount(); } return ret; } @Override public long getProactiveSpillCountObjects() { Iterator<JobStats> it = jobPlan.iterator(); long ret = 0; while (it.hasNext()) { ret += it.next().getProactiveSpillCountObjects(); } return ret; } @Override public long getProactiveSpillCountRecords() { Iterator<JobStats> it = jobPlan.iterator(); long ret = 0; while (it.hasNext()) { ret += it.next().getProactiveSpillCountRecs(); } return ret; } @Override public long getBytesWritten() { Iterator<JobStats> it = jobPlan.iterator(); long ret = 0; while (it.hasNext()) { long n = it.next().getBytesWritten(); if (n > 0) ret += n; } return ret; } @Override public long getRecordWritten() { Iterator<JobStats> it = jobPlan.iterator(); long ret = 0; while (it.hasNext()) { long n = it.next().getRecordWrittern(); if (n > 0) ret += n; } return ret; } @Override public String getScriptId() { return ScriptState.get().getId(); } @Override public String getFeatures() { return ScriptState.get().getScriptFeatures(); } @Override public long getDuration() { return (startTime > 0 && endTime > 0) ? (endTime - startTime) : -1; } @Override public int getNumberJobs() { return jobPlan.size(); } @Override public List<OutputStats> getOutputStats() { List<OutputStats> outputs = new ArrayList<OutputStats>(); Iterator<JobStats> iter = jobPlan.iterator(); while (iter.hasNext()) { for (OutputStats os : iter.next().getOutputs()) { outputs.add(os); } } return Collections.unmodifiableList(outputs); } @Override public OutputStats result(String alias) { if (aliasOuputMap == null) { aliasOuputMap = new HashMap<String, OutputStats>(); Iterator<JobStats> iter = jobPlan.iterator(); while (iter.hasNext()) { for (OutputStats os : iter.next().getOutputs()) { String a = os.getAlias(); if (a == null || a.length() == 0) { LOG.warn("Output alias isn't avalable for " + os.getLocation()); continue; } aliasOuputMap.put(a, os); } } } return aliasOuputMap.get(alias); } @Override public List<InputStats> getInputStats() { List<InputStats> inputs = new ArrayList<InputStats>(); Iterator<JobStats> iter = jobPlan.iterator(); while (iter.hasNext()) { for (InputStats is : iter.next().getInputs()) { inputs.add(is); } } return Collections.unmodifiableList(inputs); } SimplePigStats() { jobMroMap = new HashMap<Job, MapReduceOper>(); jobPlan = new JobGraph(); } void start(PigContext pigContext, JobClient jobClient, JobControlCompiler jcc, MROperPlan mrPlan) { if (pigContext == null || jobClient == null || jcc == null) { LOG.warn("invalid params: " + pigContext + jobClient + jcc); return; } this.pigContext = pigContext; this.jobClient = jobClient; this.jcc = jcc; // build job DAG with job ids assigned to null try { new JobGraphBuilder(mrPlan).visit(); } catch (VisitorException e) { LOG.warn("unable to build job plan", e); } startTime = System.currentTimeMillis(); userId = System.getProperty("user.name"); } void stop() { endTime = System.currentTimeMillis(); int m = getNumberSuccessfulJobs(); int n = getNumberFailedJobs(); if (n == 0 && m > 0 && m == jobPlan.size()) { returnCode = ReturnCode.SUCCESS; } else if (m > 0 && m < jobPlan.size()) { returnCode = ReturnCode.PARTIAL_FAILURE; } else { returnCode = ReturnCode.FAILURE; } } boolean isInitialized() { return startTime > 0; } @Override public JobClient getJobClient() { return jobClient; } JobControlCompiler getJobControlCompiler() { return jcc; } @SuppressWarnings("deprecation") JobStats addJobStats(Job job) { MapReduceOper mro = jobMroMap.get(job); if (mro == null) { LOG.warn("unable to get MR oper for job: " + job.toString()); return null; } JobStats js = mroJobMap.get(mro); JobID jobId = job.getAssignedJobID(); js.setId(jobId); js.setAlias(mro); js.setConf(job.getJobConf()); return js; } @SuppressWarnings("deprecation") public JobStats addJobStatsForNative(NativeMapReduceOper mr) { JobStats js = mroJobMap.get(mr); js.setId(new JobID(mr.getJobId(), NativeMapReduceOper.getJobNumber())); js.setAlias(mr); return js; } void display() { if (returnCode == ReturnCode.UNKNOWN) { LOG.warn("unknown return code, can't display the results"); return; } if (pigContext == null) { LOG.warn("unknown exec type, don't display the results"); return; } // currently counters are not working in local mode - see PIG-1286 ExecType execType = pigContext.getExecType(); if (execType == ExecType.LOCAL) { LOG.info("Detected Local mode. Stats reported below may be incomplete"); } SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT); StringBuilder sb = new StringBuilder(); sb.append("\nHadoopVersion\tPigVersion\tUserId\tStartedAt\tFinishedAt\tFeatures\n"); sb.append(getHadoopVersion()).append("\t").append(getPigVersion()).append("\t").append(userId).append("\t") .append(sdf.format(new Date(startTime))).append("\t").append(sdf.format(new Date(endTime))) .append("\t").append(getFeatures()).append("\n"); sb.append("\n"); if (returnCode == ReturnCode.SUCCESS) { sb.append("Success!\n"); } else if (returnCode == ReturnCode.PARTIAL_FAILURE) { sb.append("Some jobs have failed! Stop running all dependent jobs\n"); } else { sb.append("Failed!\n"); } sb.append("\n"); if (returnCode == ReturnCode.SUCCESS || returnCode == ReturnCode.PARTIAL_FAILURE) { sb.append("Job Stats (time in seconds):\n"); if (execType == ExecType.LOCAL) { sb.append(JobStats.SUCCESS_HEADER_LOCAL).append("\n"); } else { sb.append(JobStats.SUCCESS_HEADER).append("\n"); } List<JobStats> arr = jobPlan.getSuccessfulJobs(); for (JobStats js : arr) { sb.append(js.getDisplayString(execType == ExecType.LOCAL)); } sb.append("\n"); } if (returnCode == ReturnCode.FAILURE || returnCode == ReturnCode.PARTIAL_FAILURE) { sb.append("Failed Jobs:\n"); sb.append(JobStats.FAILURE_HEADER).append("\n"); List<JobStats> arr = jobPlan.getFailedJobs(); for (JobStats js : arr) { sb.append(js.getDisplayString(execType == ExecType.LOCAL)); } sb.append("\n"); } sb.append("Input(s):\n"); for (InputStats is : getInputStats()) { sb.append(is.getDisplayString(execType == ExecType.LOCAL)); } sb.append("\n"); sb.append("Output(s):\n"); for (OutputStats ds : getOutputStats()) { sb.append(ds.getDisplayString(execType == ExecType.LOCAL)); } if (execType != ExecType.LOCAL) { sb.append("\nCounters:\n"); sb.append("Total records written : " + getRecordWritten()).append("\n"); sb.append("Total bytes written : " + getBytesWritten()).append("\n"); sb.append("Spillable Memory Manager spill count : " + getSMMSpillCount()).append("\n"); sb.append("Total bags proactively spilled: " + getProactiveSpillCountObjects()).append("\n"); sb.append("Total records proactively spilled: " + getProactiveSpillCountRecords()).append("\n"); } sb.append("\nJob DAG:\n").append(jobPlan.toString()); LOG.info("Script Statistics: \n" + sb.toString()); } void mapMROperToJob(MapReduceOper mro, Job job) { if (mro == null) { LOG.warn("null MR operator"); } else { JobStats js = mroJobMap.get(mro); if (js == null) { LOG.warn("null job stats for mro: " + mro.getOperatorKey()); } else { jobMroMap.put(job, mro); } } } void setBackendException(Job job, Exception e) { if (e instanceof PigException) { LOG.error("ERROR " + ((PigException) e).getErrorCode() + ": " + e.getLocalizedMessage()); } else if (e != null) { LOG.error("ERROR: " + e.getLocalizedMessage()); } if (job.getAssignedJobID() == null || e == null) { LOG.debug("unable to set backend exception"); return; } String id = job.getAssignedJobID().toString(); Iterator<JobStats> iter = jobPlan.iterator(); while (iter.hasNext()) { JobStats js = iter.next(); if (id.equals(js.getJobId())) { js.setBackendException(e); break; } } } PigContext getPigContext() { return pigContext; } int getNumberSuccessfulJobs() { Iterator<JobStats> iter = jobPlan.iterator(); int count = 0; while (iter.hasNext()) { if (iter.next().getState() == JobState.SUCCESS) count++; } return count; } int getNumberFailedJobs() { Iterator<JobStats> iter = jobPlan.iterator(); int count = 0; while (iter.hasNext()) { if (iter.next().getState() == JobState.FAILED) count++; } return count; } boolean isJobSeen(Job job) { return !jobSeen.add(job); } }