org.apache.hama.bsp.TaskLog.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hama.bsp.TaskLog.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hama.bsp;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hama.HamaConfiguration;

/**
 * A simple logger to handle the task-specific user logs.
 */
public class TaskLog {
    private static final Log LOG = LogFactory.getLog(TaskLog.class.getName());

    private static final File LOG_DIR = new File(System.getProperty("hama.log.dir"), "tasklogs").getAbsoluteFile();

    static {
        if (!LOG_DIR.exists()) {
            LOG_DIR.mkdirs();
        }
    }

    /*
     * Get LogFile by taskid and distinguish between log and error extension
     */
    public static File getTaskLogFile(TaskAttemptID taskid, LogName filter) {
        // TODO clean up the log path and type.
        return new File(LOG_DIR,
                taskid.getJobID() + "/" + taskid.toString() + ((filter == LogName.STDERR) ? ".err" : ".log"));
    }

    /*
     * Get LogFile by stringPattern and distinguish between log and error
     * extension
     */
    public static File getLocalTaskLogFile(LogName filter, String stringPattern) {
        // TODO clean up the log path and type.
        SimpleDateFormat sdf = new SimpleDateFormat();
        sdf.applyPattern(stringPattern);
        return new File(LOG_DIR, "job_" + sdf.format(new Date()) + "/" + "local_" + sdf.format(new Date())
                + ((filter == LogName.STDERR) ? ".err" : ".log"));
    }

    /**
     * The filter for userlogs.
     */
    public static enum LogName {
        /** Log on the stdout of the task. */
        STDOUT("stdout"),

        /** Log on the stderr of the task. */
        STDERR("stderr"),

        /** Log on the map-reduce system logs of the task. */
        SYSLOG("syslog"),

        /** The java profiler information. */
        PROFILE("profile.out"),

        /** Log the debug script's stdout */
        DEBUGOUT("debugout");

        private String prefix;

        private LogName(String prefix) {
            this.prefix = prefix;
        }

        @Override
        public String toString() {
            return prefix;
        }
    }

    private static class TaskLogsPurgeFilter implements FileFilter {
        long purgeTimeStamp;

        TaskLogsPurgeFilter(long purgeTimeStamp) {
            this.purgeTimeStamp = purgeTimeStamp;
        }

        @Override
        public boolean accept(File file) {
            LOG.debug("PurgeFilter - file: " + file + ", mtime: " + file.lastModified() + ", purge: "
                    + purgeTimeStamp);
            return file.lastModified() < purgeTimeStamp;
        }
    }

    /**
     * Purge old user logs.
     * 
     * @throws IOException
     */
    public static synchronized void cleanup(int logsRetainHours) throws IOException {
        // Purge logs of tasks on this tasktracker if their
        // mtime has exceeded "bsp.task.log.retain" hours
        long purgeTimeStamp = System.currentTimeMillis() - (logsRetainHours * 60L * 60 * 1000);
        File[] oldTaskLogs = LOG_DIR.listFiles(new TaskLogsPurgeFilter(purgeTimeStamp));
        if (oldTaskLogs != null) {
            for (File oldTaskLog : oldTaskLogs) {
                FileUtil.fullyDelete(oldTaskLog);
            }
        }
    }

    static class Reader extends InputStream {
        private long bytesRemaining;
        private FileInputStream file;

        /**
         * Read a log file from start to end positions. The offsets may be negative,
         * in which case they are relative to the end of the file. For example,
         * Reader(taskid, kind, 0, -1) is the entire file and Reader(taskid, kind,
         * -4197, -1) is the last 4196 bytes.
         * 
         * @param taskid the id of the task to read the log file for
         * @param kind the kind of log to read
         * @param pStart the offset to read from (negative is relative to tail)
         * @param pEnd the offset to read upto (negative is relative to tail)
         * @throws IOException
         */
        public Reader(TaskAttemptID taskid, LogName kind, long pStart, long pEnd) throws IOException {
            long start = pStart;
            long end = pEnd;
            // find the right log file
            File filename = getTaskLogFile(taskid, kind);
            // calculate the start and stop
            long size = filename.length();
            if (start < 0) {
                start += size + 1;
            }
            if (end < 0) {
                end += size + 1;
            }
            start = Math.max(0, Math.min(start, size));
            end = Math.max(0, Math.min(end, size));
            bytesRemaining = end - start;
            file = new FileInputStream(filename);
            // skip upto start
            long pos = 0;
            while (pos < start) {
                long result = file.skip(start - pos);
                if (result < 0) {
                    bytesRemaining = 0;
                    break;
                }
                pos += result;
            }
        }

        @Override
        public int read() throws IOException {
            int result = -1;
            if (bytesRemaining > 0) {
                bytesRemaining -= 1;
                result = file.read();
            }
            return result;
        }

        @Override
        public int read(byte[] buffer, int offset, int pLength) throws IOException {
            int length = (int) Math.min(pLength, bytesRemaining);
            int bytes = file.read(buffer, offset, length);
            if (bytes > 0) {
                bytesRemaining -= bytes;
            }
            return bytes;
        }

        @Override
        public int available() throws IOException {
            return (int) Math.min(bytesRemaining, file.available());
        }

        @Override
        public void close() throws IOException {
            file.close();
        }
    }

    private static final String bashCommand = "bash";
    private static final String tailCommand = "tail";

    /**
     * Get the desired maximum length of task's logs.
     * 
     * @param conf the job to look in
     * @return the number of bytes to cap the log files at
     */
    public static long getTaskLogLength(HamaConfiguration conf) {
        return conf.getLong("bsp.userlog.limit.kb", 100) * 1024;
    }

    /**
     * Wrap a command in a shell to capture stdout and stderr to files. If the
     * tailLength is 0, the entire output will be saved.
     * 
     * @param cmd The command and the arguments that should be run
     * @param stdoutFilename The filename that stdout should be saved to
     * @param stderrFilename The filename that stderr should be saved to
     * @param tailLength The length of the tail to be saved.
     * @return the modified command that should be run
     */
    public static List<String> captureOutAndError(List<String> cmd, File stdoutFilename, File stderrFilename,
            long tailLength) throws IOException {
        return captureOutAndError(null, cmd, stdoutFilename, stderrFilename, tailLength);
    }

    /**
     * Wrap a command in a shell to capture stdout and stderr to files. Setup
     * commands such as setting memory limit can be passed which will be executed
     * before exec. If the tailLength is 0, the entire output will be saved.
     * 
     * @param setup The setup commands for the execed process.
     * @param cmd The command and the arguments that should be run
     * @param stdoutFilename The filename that stdout should be saved to
     * @param stderrFilename The filename that stderr should be saved to
     * @param tailLength The length of the tail to be saved.
     * @return the modified command that should be run
     */
    public static List<String> captureOutAndError(List<String> setup, List<String> cmd, File stdoutFilename,
            File stderrFilename, long tailLength) throws IOException {
        String stdout = FileUtil.makeShellPath(stdoutFilename);
        String stderr = FileUtil.makeShellPath(stderrFilename);
        List<String> result = new ArrayList<String>(3);
        result.add(bashCommand);
        result.add("-c");
        StringBuffer mergedCmd = new StringBuffer();
        if (setup != null && setup.size() > 0) {
            mergedCmd.append(addCommand(setup, false));
            mergedCmd.append(";");
        }
        if (tailLength > 0) {
            mergedCmd.append("(");
        } else {
            mergedCmd.append("exec ");
        }
        mergedCmd.append(addCommand(cmd, true));
        mergedCmd.append(" < /dev/null ");
        if (tailLength > 0) {
            mergedCmd.append(" | ");
            mergedCmd.append(tailCommand);
            mergedCmd.append(" -c ");
            mergedCmd.append(tailLength);
            mergedCmd.append(" > ");
            mergedCmd.append(stdout);
            mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
            mergedCmd.append(tailCommand);
            mergedCmd.append(" -c ");
            mergedCmd.append(tailLength);
            mergedCmd.append(" > ");
            mergedCmd.append(stderr);
            mergedCmd.append(" ; exit $PIPESTATUS");
        } else {
            mergedCmd.append(" 1> ");
            mergedCmd.append(stdout);
            mergedCmd.append(" 2> ");
            mergedCmd.append(stderr);
        }
        result.add(mergedCmd.toString());
        return result;
    }

    public static List<String> captureOutAndErrorTee(List<String> setup, List<String> cmd, File stdoutFilename,
            File stderrFilename, long tailLength) throws IOException {
        String stdout = FileUtil.makeShellPath(stdoutFilename);
        List<String> result = new ArrayList<String>(3);
        result.add(bashCommand);
        result.add("-c");
        StringBuilder mergedCmd = new StringBuilder();

        mergedCmd.append(addCommand(cmd, true));
        mergedCmd.append(" 2>&1 | tee ").append(stdout);

        result.add(mergedCmd.toString());
        return result;
    }

    /**
     * Add quotes to each of the command strings and return as a single string
     * 
     * @param cmd The command to be quoted
     * @param pIsExecutable makes shell path if the first argument is executable
     * @return returns The quoted string.
     * @throws IOException
     */
    public static String addCommand(List<String> cmd, boolean pIsExecutable) throws IOException {
        boolean isExecutable = pIsExecutable;
        StringBuffer command = new StringBuffer();
        for (String s : cmd) {
            command.append('\'');
            if (isExecutable) {
                // the executable name needs to be expressed as a shell path for the
                // shell to find it.
                command.append(FileUtil.makeShellPath(new File(s)));
                isExecutable = false;
            } else {
                command.append(s);
            }
            command.append('\'');
            command.append(" ");
        }
        return command.toString();
    }

    /**
     * Wrap a command in a shell to capture debug script's stdout and stderr to
     * debugout.
     * 
     * @param cmd The command and the arguments that should be run
     * @param debugoutFilename The filename that stdout and stderr should be saved
     *          to.
     * @return the modified command that should be run
     * @throws IOException
     */
    public static List<String> captureDebugOut(List<String> cmd, File debugoutFilename) throws IOException {
        String debugout = FileUtil.makeShellPath(debugoutFilename);
        List<String> result = new ArrayList<String>(3);
        result.add(bashCommand);
        result.add("-c");
        StringBuffer mergedCmd = new StringBuffer();
        mergedCmd.append("exec ");
        boolean isExecutable = true;
        for (String s : cmd) {
            if (isExecutable) {
                // the executable name needs to be expressed as a shell path for the
                // shell to find it.
                mergedCmd.append(FileUtil.makeShellPath(new File(s)));
                isExecutable = false;
            } else {
                mergedCmd.append(s);
            }
            mergedCmd.append(" ");
        }
        mergedCmd.append(" < /dev/null ");
        mergedCmd.append(" >");
        mergedCmd.append(debugout);
        mergedCmd.append(" 2>&1 ");
        result.add(mergedCmd.toString());
        return result;
    }

    /**
     * Get the desired maximum length of task's logs.
     * 
     * @param conf the job to look in
     * @return the number of bytes to cap the log files at
     */
    public static long getTaskLogLength(Configuration conf) {
        return conf.getLong("bsp.userlog.limit.kb", 100) * 1024;
    }

} // TaskLog