Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskReport; import org.apache.hadoop.mapred.jobcontrol.Job; import org.apache.hadoop.mapred.jobcontrol.JobControl; import org.apache.pig.FuncSpec; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.plan.PlanException; import org.apache.pig.impl.plan.VisitorException; import org.apache.pig.impl.util.LogUtils; import org.apache.pig.tools.pigstats.PigStats; public abstract class Launcher { private static final Log log = LogFactory.getLog(Launcher.class); long totalHadoopTimeSpent; String newLine = "\n"; boolean pigException = false; boolean outOfMemory = false; static final String OOM_ERR = "OutOfMemoryError"; protected Launcher() { totalHadoopTimeSpent = 0; //handle the windows portion of \r if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")) { newLine = "\r\n"; } reset(); } /** * Resets the state after a launch */ public void reset() { } /** * Method to launch pig for hadoop either for a cluster's * job tracker or for a local job runner. THe only difference * between the two is the job client. Depending on the pig context * the job client will be initialize to one of the two. * Launchers for other frameworks can overide these methods. * Given an input PhysicalPlan, it compiles it * to get a MapReduce Plan. The MapReduce plan which * has multiple MapReduce operators each one of which * has to be run as a map reduce job with dependency * information stored in the plan. It compiles the * MROperPlan into a JobControl object. Each Map Reduce * operator is converted into a Job and added to the JobControl * object. Each Job also has a set of dependent Jobs that * are created using the MROperPlan. * The JobControl object is obtained from the JobControlCompiler * Then a new thread is spawned that submits these jobs * while respecting the dependency information. * The parent thread monitors the submitted jobs' progress and * after it is complete, stops the JobControl thread. * @param php * @param grpName * @param pc * @throws PlanException * @throws VisitorException * @throws IOException * @throws ExecException * @throws JobCreationException */ public abstract PigStats launchPig(PhysicalPlan php, String grpName, PigContext pc) throws PlanException, VisitorException, IOException, ExecException, JobCreationException, Exception; /** * Explain how a pig job will be executed on the underlying * infrastructure. * @param pp PhysicalPlan to explain * @param pc PigContext to use for configuration * @param ps PrintStream to write output on. * @param format Format to write in * @param verbose Amount of information to print * @throws VisitorException * @throws IOException */ public abstract void explain(PhysicalPlan pp, PigContext pc, PrintStream ps, String format, boolean verbose) throws PlanException, VisitorException, IOException; protected boolean isComplete(double prog) { return (int) (Math.ceil(prog)) == 1; } protected void getStats(Job job, JobClient jobClient, boolean errNotDbg, PigContext pigContext) throws Exception { JobID MRJobID = job.getAssignedJobID(); String jobMessage = job.getMessage(); Exception backendException = null; if (MRJobID == null) { try { LogUtils.writeLog("Backend error message during job submission", jobMessage, pigContext.getProperties().getProperty("pig.logfile"), log); backendException = getExceptionFromString(jobMessage); } catch (Exception e) { int errCode = 2997; String msg = "Unable to recreate exception from backend error: " + jobMessage; throw new ExecException(msg, errCode, PigException.BUG); } throw backendException; } try { TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID); getErrorMessages(mapRep, "map", errNotDbg, pigContext); totalHadoopTimeSpent += computeTimeSpent(mapRep); mapRep = null; TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID); getErrorMessages(redRep, "reduce", errNotDbg, pigContext); totalHadoopTimeSpent += computeTimeSpent(redRep); redRep = null; } catch (IOException e) { if (job.getState() == Job.SUCCESS) { // if the job succeeded, let the user know that // we were unable to get statistics log.warn("Unable to get job related diagnostics"); } else { throw e; } } } protected long computeTimeSpent(TaskReport[] taskReports) { long timeSpent = 0; for (TaskReport r : taskReports) { timeSpent += (r.getFinishTime() - r.getStartTime()); } return timeSpent; } protected void getErrorMessages(TaskReport reports[], String type, boolean errNotDbg, PigContext pigContext) throws Exception { for (int i = 0; i < reports.length; i++) { String msgs[] = reports[i].getDiagnostics(); ArrayList<Exception> exceptions = new ArrayList<Exception>(); String exceptionCreateFailMsg = null; boolean jobFailed = false; float successfulProgress = 1.0f; if (msgs.length > 0) { //if the progress reported is not 1.0f then the map or reduce job failed //this comparison is in place till Hadoop 0.20 provides methods to query //job status if (reports[i].getProgress() != successfulProgress) { jobFailed = true; } Set<String> errorMessageSet = new HashSet<String>(); for (int j = 0; j < msgs.length; j++) { if (!errorMessageSet.contains(msgs[j])) { errorMessageSet.add(msgs[j]); if (errNotDbg) { //errNotDbg is used only for failed jobs //keep track of all the unique exceptions try { LogUtils.writeLog("Backend error message", msgs[j], pigContext.getProperties().getProperty("pig.logfile"), log); Exception e = getExceptionFromString(msgs[j]); exceptions.add(e); } catch (Exception e1) { exceptionCreateFailMsg = msgs[j]; } } else { log.debug("Error message from task (" + type + ") " + reports[i].getTaskID() + msgs[j]); } } } } //if there are no valid exception that could be created, report if ((exceptions.size() == 0) && (exceptionCreateFailMsg != null)) { int errCode = 2997; String msg = "Unable to recreate exception from backed error: " + exceptionCreateFailMsg; throw new ExecException(msg, errCode, PigException.BUG); } //if its a failed job then check if there is more than one exception //more than one exception implies possibly different kinds of failures //log all the different failures and throw the exception corresponding //to the first failure if (jobFailed) { if (exceptions.size() > 1) { for (int j = 0; j < exceptions.size(); ++j) { String headerMessage = "Error message from task (" + type + ") " + reports[i].getTaskID(); LogUtils.writeLog(exceptions.get(j), pigContext.getProperties().getProperty("pig.logfile"), log, false, headerMessage, false, false); } throw exceptions.get(0); } else if (exceptions.size() == 1) { throw exceptions.get(0); } else { int errCode = 2115; String msg = "Internal error. Expected to throw exception from the backend. Did not find any exception to throw."; throw new ExecException(msg, errCode, PigException.BUG); } } } } /** * Compute the progress of the current job submitted * through the JobControl object jc to the JobClient jobClient * @param jc - The JobControl object that has been submitted * @param jobClient - The JobClient to which it has been submitted * @return The progress as a precentage in double format * @throws IOException */ protected double calculateProgress(JobControl jc, JobClient jobClient) throws IOException { double prog = 0.0; prog += jc.getSuccessfulJobs().size(); List runnJobs = jc.getRunningJobs(); for (Object object : runnJobs) { Job j = (Job) object; prog += progressOfRunningJob(j, jobClient); } return prog; } /** * Returns the progress of a Job j which is part of a submitted * JobControl object. The progress is for this Job. So it has to * be scaled down by the num of jobs that are present in the * JobControl. * @param j - The Job for which progress is required * @param jobClient - the JobClient to which it has been submitted * @return Returns the percentage progress of this Job * @throws IOException */ protected double progressOfRunningJob(Job j, JobClient jobClient) throws IOException { JobID mrJobID = j.getAssignedJobID(); RunningJob rj = jobClient.getJob(mrJobID); if (rj == null && j.getState() == Job.SUCCESS) return 1; else if (rj == null) return 0; else { double mapProg = rj.mapProgress(); double redProg = rj.reduceProgress(); return (mapProg + redProg) / 2; } } public long getTotalHadoopTimeSpent() { return totalHadoopTimeSpent; } /** * * @param stackTraceLine The string representation of {@link Throwable#printStackTrace() printStackTrace} * Handles internal PigException and its subclasses that override the {@link Throwable#toString() toString} method * @return An exception object whose string representation of printStackTrace is the input stackTrace * @throws Exception */ Exception getExceptionFromString(String stackTrace) throws Exception { String[] lines = stackTrace.split(newLine); Throwable t = getExceptionFromStrings(lines, 0); if (!pigException) { int errCode = 6015; String msg = "During execution, encountered a Hadoop error."; ExecException ee = new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, t); ee.setStackTrace(t.getStackTrace()); return ee; } else { pigException = false; if (outOfMemory) { outOfMemory = false; int errCode = 6016; String msg = "Out of memory."; ExecException ee = new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, t); ee.setStackTrace(t.getStackTrace()); return ee; } return (Exception) t; } } /** * * @param stackTraceLine An array of strings that represent {@link Throwable#printStackTrace() printStackTrace} * output, split by newline * @return An exception object whose string representation of printStackTrace is the input stackTrace * @throws Exception */ private Throwable getExceptionFromStrings(String[] stackTraceLines, int startingLineNum) throws Exception { /* * parse the array of string and throw the appropriate exception * first: from the line startingLineNum extract the exception name extract the message if any * fourth: create the appropriate exception and return it * An example of the stack trace: org.apache.pig.backend.executionengine.ExecException: ERROR 1075: Received a bytearray from the UDF. Cannot determine how to convert the bytearray to int. at org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast.getNext(POCast.java:152) at org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LessThanExpr.getNext(LessThanExpr.java:85) at org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter.getNext(POFilter.java:148) at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase.runPipeline(PigMapBase.java:184) at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase.map(PigMapBase.java:174) at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapOnly$Map.map(PigMapOnly.java:65) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:47) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:227) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2207) */ int prevStartingLineNum = startingLineNum; if (stackTraceLines.length > 0 && startingLineNum < (stackTraceLines.length - 1)) { //the regex for matching the exception class name; note the use of the $ for matching nested classes String exceptionNameDelimiter = "(\\w+(\\$\\w+)?\\.)+\\w+"; Pattern exceptionNamePattern = Pattern.compile(exceptionNameDelimiter); //from the first line extract the exception name and the exception message Matcher exceptionNameMatcher = exceptionNamePattern.matcher(stackTraceLines[startingLineNum]); String exceptionName = null; String exceptionMessage = null; if (exceptionNameMatcher.find()) { exceptionName = exceptionNameMatcher.group(); /* * note that the substring is from end + 2 * the regex matcher ends at one position beyond the match * in this case it will end at colon (:) * the exception message will have a preceding space (after the colon (:)) */ if (exceptionName.contains(OOM_ERR)) { outOfMemory = true; } if (stackTraceLines[startingLineNum].length() > exceptionNameMatcher.end()) { exceptionMessage = stackTraceLines[startingLineNum].substring(exceptionNameMatcher.end() + 2); } ++startingLineNum; } //the exceptionName should not be null if (exceptionName != null) { ArrayList<StackTraceElement> stackTraceElements = new ArrayList<StackTraceElement>(); //Create stack trace elements for the remaining lines String stackElementRegex = "\\s+at\\s+(\\w+(\\$\\w+)?\\.)+(\\<)?\\w+(\\>)?"; Pattern stackElementPattern = Pattern.compile(stackElementRegex); String pigExceptionRegex = "org\\.apache\\.pig\\."; Pattern pigExceptionPattern = Pattern.compile(pigExceptionRegex); String moreElementRegex = "\\s+\\.\\.\\.\\s+\\d+\\s+more"; Pattern moreElementPattern = Pattern.compile(moreElementRegex); String pigPackageRegex = "org.apache.pig"; int lineNum = startingLineNum; for (; lineNum < (stackTraceLines.length - 1); ++lineNum) { Matcher stackElementMatcher = stackElementPattern.matcher(stackTraceLines[lineNum]); if (stackElementMatcher.find()) { StackTraceElement ste = getStackTraceElement(stackTraceLines[lineNum]); stackTraceElements.add(ste); String className = ste.getClassName(); Matcher pigExceptionMatcher = pigExceptionPattern.matcher(className); if (pigExceptionMatcher.find()) { pigException = true; } } else { Matcher moreElementMatcher = moreElementPattern.matcher(stackTraceLines[lineNum]); if (moreElementMatcher.find()) { ++lineNum; } break; } } startingLineNum = lineNum; //create the appropriate exception; setup the stack trace and message Object object = PigContext.instantiateFuncFromSpec(exceptionName); if (object instanceof PigException) { //extract the error code and message the regex for matching the custom format of ERROR <ERROR CODE>: String errMessageRegex = "ERROR\\s+\\d+:"; Pattern errMessagePattern = Pattern.compile(errMessageRegex); Matcher errMessageMatcher = errMessagePattern.matcher(exceptionMessage); if (errMessageMatcher.find()) { String errMessageStub = errMessageMatcher.group(); /* * extract the actual exception message sans the ERROR <ERROR CODE>: * again note that the matcher ends at the space following the colon (:) * the exception message appears after the space and hence the end + 1 */ exceptionMessage = exceptionMessage.substring(errMessageMatcher.end() + 1); //the regex to match the error code wich is a string of numerals String errCodeRegex = "\\d+"; Pattern errCodePattern = Pattern.compile(errCodeRegex); Matcher errCodeMatcher = errCodePattern.matcher(errMessageStub); String code = null; if (errCodeMatcher.find()) { code = errCodeMatcher.group(); } //could receive a number format exception here but it will be propagated up the stack int errCode; if (code != null) errCode = Integer.parseInt(code); else errCode = 2998; //create the exception with the message and then set the error code and error source FuncSpec funcSpec = new FuncSpec(exceptionName, exceptionMessage); object = PigContext.instantiateFuncFromSpec(funcSpec); ((PigException) object).setErrorCode(errCode); ((PigException) object).setErrorSource(PigException.determineErrorSource(errCode)); } else { //else for if(errMessageMatcher.find()) /* * did not find the error code which means that the PigException or its * subclass is not returning the error code * highly unlikely: should never be here */ FuncSpec funcSpec = new FuncSpec(exceptionName, exceptionMessage); object = PigContext.instantiateFuncFromSpec(funcSpec); ((PigException) object).setErrorCode(2997);//generic error code ((PigException) object).setErrorSource(PigException.BUG); } } else { //else for if(object instanceof PigException) //its not PigException; create the exception with the message object = PigContext.instantiateFuncFromSpec(new FuncSpec(exceptionName, exceptionMessage)); } StackTraceElement[] steArr = new StackTraceElement[stackTraceElements.size()]; ((Throwable) object).setStackTrace(stackTraceElements.toArray(steArr)); if (startingLineNum < (stackTraceLines.length - 1)) { Throwable e = getExceptionFromStrings(stackTraceLines, startingLineNum); ((Throwable) object).initCause(e); } return (Throwable) object; } else { //else for if(exceptionName != null) int errCode = 2055; String msg = "Did not find exception name to create exception from string: " + Arrays.toString(stackTraceLines); throw new ExecException(msg, errCode, PigException.BUG); } } else { //else for if(lines.length > 0) int errCode = 2056; String msg = "Cannot create exception from empty string."; throw new ExecException(msg, errCode, PigException.BUG); } } /** * * @param line the string representation of a stack trace returned by {@link Throwable#printStackTrace() printStackTrace} * @return the StackTraceElement object representing the stack trace * @throws Exception */ public StackTraceElement getStackTraceElement(String line) throws Exception { /* * the format of the line is something like: * at org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapOnly$Map.map(PigMapOnly.java:65) * note the white space before the 'at'. Its not of much importance but noted for posterity. */ String[] items; /* * regex for matching the fully qualified method Name * note the use of the $ for matching nested classes * and the use of < and > for constructors */ String qualifiedMethodNameRegex = "(\\w+(\\$\\w+)?\\.)+(<)?\\w+(>)?"; Pattern qualifiedMethodNamePattern = Pattern.compile(qualifiedMethodNameRegex); Matcher contentMatcher = qualifiedMethodNamePattern.matcher(line); //org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapOnly$Map.map(PigMapOnly.java:65) String content = null; if (contentMatcher.find()) { content = line.substring(contentMatcher.start()); } else { int errCode = 2057; String msg = "Did not find fully qualified method name to reconstruct stack trace: " + line; throw new ExecException(msg, errCode, PigException.BUG); } Matcher qualifiedMethodNameMatcher = qualifiedMethodNamePattern.matcher(content); //org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapOnly$Map.map String qualifiedMethodName = null; //(PigMapOnly.java:65) String fileDetails = null; if (qualifiedMethodNameMatcher.find()) { qualifiedMethodName = qualifiedMethodNameMatcher.group(); fileDetails = content.substring(qualifiedMethodNameMatcher.end() + 1); } else { int errCode = 2057; String msg = "Did not find fully qualified method name to reconstruct stack trace: " + line; throw new ExecException(msg, errCode, PigException.BUG); } //From the fully qualified method name, extract the declaring class and method name items = qualifiedMethodName.split("\\."); //initialize the declaringClass (to org in most cases) String declaringClass = items[0]; //the last member is always the method name String methodName = items[items.length - 1]; StringBuilder sb = new StringBuilder(); //concatenate the names by adding the dot (.) between the members till the penultimate member for (int i = 1; i < items.length - 1; ++i) { sb.append('.'); sb.append(items[i]); } declaringClass += sb.toString(); //from the file details extract the file name and the line number //PigMapOnly.java:65 fileDetails = fileDetails.substring(0, fileDetails.length() - 1); items = fileDetails.split(":"); //PigMapOnly.java String fileName = null; int lineNumber = -1; if (items.length > 0) { fileName = items[0]; if (items.length > 1) { lineNumber = Integer.parseInt(items[1]); } } return new StackTraceElement(declaringClass, methodName, fileName, lineNumber); } }