Example usage for org.apache.hadoop.fs FileUtil makeShellPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil makeShellPath.

Prototype

public static String makeShellPath(File file) throws IOException 

Source Link

Document

Convert a os-native filename to a path that works for the shell.

Usage

From source file:it.crs4.pydoop.mapreduce.pipes.TaskLog.java

License:Apache License

/**
 * Construct the command line for running the task JVM
 * @param setup The setup commands for the execed process.
 * @param cmd The command and the arguments that should be run
 * @param stdoutFilename The filename that stdout should be saved to
 * @param stderrFilename The filename that stderr should be saved to
 * @param tailLength The length of the tail to be saved.
 * @return the command line as a String/*from w  ww. j  av  a 2s. c o  m*/
 * @throws IOException
 */
static String buildCommandLine(List<String> setup, List<String> cmd, File stdoutFilename, File stderrFilename,
        long tailLength, boolean useSetsid) throws IOException {

    String stdout = FileUtil.makeShellPath(stdoutFilename);
    String stderr = FileUtil.makeShellPath(stderrFilename);
    StringBuffer mergedCmd = new StringBuffer();

    // Export the pid of taskJvm to env variable JVM_PID.
    // Currently pid is not used on Windows
    if (!Shell.WINDOWS) {
        mergedCmd.append(" export JVM_PID=`echo $$` ; ");
    }

    if (setup != null && setup.size() > 0) {
        mergedCmd.append(addCommand(setup, false));
        mergedCmd.append(";");
    }
    if (tailLength > 0) {
        mergedCmd.append("(");
    } else if (ProcessTree.isSetsidAvailable && useSetsid && !Shell.WINDOWS) {
        mergedCmd.append("exec setsid ");
    } else {
        mergedCmd.append("exec ");
    }
    mergedCmd.append(addCommand(cmd, true));
    mergedCmd.append(" < /dev/null ");
    if (tailLength > 0) {
        mergedCmd.append(" | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" >> ");
        mergedCmd.append(stdout);
        mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" >> ");
        mergedCmd.append(stderr);
        mergedCmd.append(" ; exit $PIPESTATUS");
    } else {
        mergedCmd.append(" 1>> ");
        mergedCmd.append(stdout);
        mergedCmd.append(" 2>> ");
        mergedCmd.append(stderr);
    }
    return mergedCmd.toString();
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.TaskLog.java

License:Apache License

/**
 * Construct the command line for running the debug script
 * @param cmd The command and the arguments that should be run
 * @param stdoutFilename The filename that stdout should be saved to
 * @param stderrFilename The filename that stderr should be saved to
 * @param tailLength The length of the tail to be saved.
 * @return the command line as a String//  w w  w . j a  va  2  s.  co m
 * @throws IOException
 */
static String buildDebugScriptCommandLine(List<String> cmd, String debugout) throws IOException {
    StringBuilder mergedCmd = new StringBuilder();
    mergedCmd.append("exec ");
    boolean isExecutable = true;
    for (String s : cmd) {
        if (isExecutable) {
            // the executable name needs to be expressed as a shell path for the  
            // shell to find it.
            mergedCmd.append(FileUtil.makeShellPath(new File(s)));
            isExecutable = false;
        } else {
            mergedCmd.append(s);
        }
        mergedCmd.append(" ");
    }
    mergedCmd.append(" < /dev/null ");
    mergedCmd.append(" >");
    mergedCmd.append(debugout);
    mergedCmd.append(" 2>&1 ");
    return mergedCmd.toString();
}

---

From source file:it.crs4.pydoop.mapreduce.pipes.TaskLog.java

License:Apache License

/**
 * Add quotes to each of the command strings and
 * return as a single string // w w w.jav  a2  s.  c  o  m
 * @param cmd The command to be quoted
 * @param isExecutable makes shell path if the first 
 * argument is executable
 * @return returns The quoted string. 
 * @throws IOException
 */
public static String addCommand(List<String> cmd, boolean isExecutable) throws IOException {
    StringBuffer command = new StringBuffer();
    for (String s : cmd) {
        command.append('\'');
        if (isExecutable) {
            // the executable name needs to be expressed as a shell path for the  
            // shell to find it.
            command.append(FileUtil.makeShellPath(new File(s)));
            isExecutable = false;
        } else {
            command.append(s);
        }
        command.append('\'');
        command.append(" ");
    }
    return command.toString();
}

---

From source file:org.apache.hadoop.fs.FileUtil.java

License:Apache License

/**
 * Given a Tar File as input it will untar the file in a the untar directory
 * passed as the second parameter//from w w w. j  a v a  2 s.  c  om
 *
 * This utility will untar ".tar" files and ".tar.gz","tgz" files.
 *
 * @param inFile The tar file as input.
 * @param untarDir The untar directory where to untar the tar file.
 * @throws java.io.IOException
 */
public static void unTar(File inFile, File untarDir) throws IOException {
    if (!untarDir.mkdirs()) {
        if (!untarDir.isDirectory()) {
            throw new IOException("Mkdirs failed to create " + untarDir);
        }
    }

    StringBuffer untarCommand = new StringBuffer();
    boolean gzipped = inFile.toString().endsWith("gz");
    if (gzipped) {
        untarCommand.append(" gzip -dc '");
        untarCommand.append(FileUtil.makeShellPath(inFile));
        untarCommand.append("' | (");
    }
    untarCommand.append("cd '");
    untarCommand.append(FileUtil.makeShellPath(untarDir));
    untarCommand.append("' ; ");
    untarCommand.append("tar -xf ");

    if (gzipped) {
        untarCommand.append(" -)");
    } else {
        untarCommand.append(FileUtil.makeShellPath(inFile));
    }
    String[] shellCmd = { "bash", "-c", untarCommand.toString() };
    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
    shexec.execute();
    int exitcode = shexec.getExitCode();
    if (exitcode != 0) {
        throw new IOException(
                "Error untarring file " + inFile + ". Tar process exited with exit code " + exitcode);
    }
}

---

From source file:org.apache.hama.bsp.TaskLog.java

License:Apache License

/**
 * Wrap a command in a shell to capture stdout and stderr to files. Setup
 * commands such as setting memory limit can be passed which will be executed
 * before exec. If the tailLength is 0, the entire output will be saved.
 * //from  w w  w  . j  ava2  s .  co m
 * @param setup The setup commands for the execed process.
 * @param cmd The command and the arguments that should be run
 * @param stdoutFilename The filename that stdout should be saved to
 * @param stderrFilename The filename that stderr should be saved to
 * @param tailLength The length of the tail to be saved.
 * @return the modified command that should be run
 */
public static List<String> captureOutAndError(List<String> setup, List<String> cmd, File stdoutFilename,
        File stderrFilename, long tailLength) throws IOException {
    String stdout = FileUtil.makeShellPath(stdoutFilename);
    String stderr = FileUtil.makeShellPath(stderrFilename);
    List<String> result = new ArrayList<String>(3);
    result.add(bashCommand);
    result.add("-c");
    StringBuffer mergedCmd = new StringBuffer();
    if (setup != null && setup.size() > 0) {
        mergedCmd.append(addCommand(setup, false));
        mergedCmd.append(";");
    }
    if (tailLength > 0) {
        mergedCmd.append("(");
    } else {
        mergedCmd.append("exec ");
    }
    mergedCmd.append(addCommand(cmd, true));
    mergedCmd.append(" < /dev/null ");
    if (tailLength > 0) {
        mergedCmd.append(" | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" > ");
        mergedCmd.append(stdout);
        mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" > ");
        mergedCmd.append(stderr);
        mergedCmd.append(" ; exit $PIPESTATUS");
    } else {
        mergedCmd.append(" 1> ");
        mergedCmd.append(stdout);
        mergedCmd.append(" 2> ");
        mergedCmd.append(stderr);
    }
    result.add(mergedCmd.toString());
    return result;
}

---

From source file:org.apache.hama.bsp.TaskLog.java

License:Apache License

public static List<String> captureOutAndErrorTee(List<String> setup, List<String> cmd, File stdoutFilename,
        File stderrFilename, long tailLength) throws IOException {
    String stdout = FileUtil.makeShellPath(stdoutFilename);
    List<String> result = new ArrayList<String>(3);
    result.add(bashCommand);//from  w w w  . j  a v a 2 s. co m
    result.add("-c");
    StringBuilder mergedCmd = new StringBuilder();

    mergedCmd.append(addCommand(cmd, true));
    mergedCmd.append(" 2>&1 | tee ").append(stdout);

    result.add(mergedCmd.toString());
    return result;
}

---

From source file:org.apache.hama.bsp.TaskLog.java

License:Apache License

/**
 * Add quotes to each of the command strings and return as a single string
 * /*  w  w  w  .ja v a 2  s .c  om*/
 * @param cmd The command to be quoted
 * @param pIsExecutable makes shell path if the first argument is executable
 * @return returns The quoted string.
 * @throws IOException
 */
public static String addCommand(List<String> cmd, boolean pIsExecutable) throws IOException {
    boolean isExecutable = pIsExecutable;
    StringBuffer command = new StringBuffer();
    for (String s : cmd) {
        command.append('\'');
        if (isExecutable) {
            // the executable name needs to be expressed as a shell path for the
            // shell to find it.
            command.append(FileUtil.makeShellPath(new File(s)));
            isExecutable = false;
        } else {
            command.append(s);
        }
        command.append('\'');
        command.append(" ");
    }
    return command.toString();
}

---

From source file:org.apache.hama.bsp.TaskLog.java

License:Apache License

/**
 * Wrap a command in a shell to capture debug script's stdout and stderr to
 * debugout.//from w ww . j  a v  a  2s  .  co m
 * 
 * @param cmd The command and the arguments that should be run
 * @param debugoutFilename The filename that stdout and stderr should be saved
 *          to.
 * @return the modified command that should be run
 * @throws IOException
 */
public static List<String> captureDebugOut(List<String> cmd, File debugoutFilename) throws IOException {
    String debugout = FileUtil.makeShellPath(debugoutFilename);
    List<String> result = new ArrayList<String>(3);
    result.add(bashCommand);
    result.add("-c");
    StringBuffer mergedCmd = new StringBuffer();
    mergedCmd.append("exec ");
    boolean isExecutable = true;
    for (String s : cmd) {
        if (isExecutable) {
            // the executable name needs to be expressed as a shell path for the
            // shell to find it.
            mergedCmd.append(FileUtil.makeShellPath(new File(s)));
            isExecutable = false;
        } else {
            mergedCmd.append(s);
        }
        mergedCmd.append(" ");
    }
    mergedCmd.append(" < /dev/null ");
    mergedCmd.append(" >");
    mergedCmd.append(debugout);
    mergedCmd.append(" 2>&1 ");
    result.add(mergedCmd.toString());
    return result;
}

---

From source file:org.apache.hama.myhama.util.TaskLog.java

License:Apache License

/**
 * Wrap a command in a shell to capture stdout and stderr to files. Setup
 * commands such as setting memory limit can be passed which will be executed
 * before exec. If the tailLength is 0, the entire output will be saved.
 * //from  ww  w .j  a  v  a2s.co  m
 * @param setup The setup commands for the execed process.
 * @param cmd The command and the arguments that should be run
 * @param stdoutFilename The filename that stdout should be saved to
 * @param stderrFilename The filename that stderr should be saved to
 * @param tailLength The length of the tail to be saved.
 * @return the modified command that should be run
 */
public static List<String> captureOutAndError(List<String> setup, List<String> cmd, File stdoutFilename,
        File stderrFilename, long tailLength) throws IOException {
    String stdout = FileUtil.makeShellPath(stdoutFilename);
    String stderr = FileUtil.makeShellPath(stderrFilename);
    List<String> result = new ArrayList<String>(3);
    result.add(bashCommand);
    result.add("-c");
    StringBuffer mergedCmd = new StringBuffer();
    if (setup != null && setup.size() > 0) {
        mergedCmd.append(addCommand(setup, false));
        mergedCmd.append(";");
    }
    if (tailLength > 0) {
        mergedCmd.append("(");
    } else {
        mergedCmd.append("exec ");
    }
    mergedCmd.append(addCommand(cmd, true));
    mergedCmd.append(" < /dev/null ");
    if (tailLength > 0) {
        mergedCmd.append(" | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" >> ");
        mergedCmd.append(stdout);
        mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
        mergedCmd.append(tailCommand);
        mergedCmd.append(" -c ");
        mergedCmd.append(tailLength);
        mergedCmd.append(" >> ");
        mergedCmd.append(stderr);
        mergedCmd.append(" ; exit $PIPESTATUS");
    } else {
        mergedCmd.append(" 1>> ");
        mergedCmd.append(stdout);
        mergedCmd.append(" 2>> ");
        mergedCmd.append(stderr);
    }
    result.add(mergedCmd.toString());
    return result;
}

---

From source file:org.apache.hama.myhama.util.TaskLog.java

License:Apache License

/**
 * Add quotes to each of the command strings and return as a single string
 * /*from   w w w. j a  v  a2s  .c om*/
 * @param cmd The command to be quoted
 * @param isExecutable makes shell path if the first argument is executable
 * @return returns The quoted string.
 * @throws IOException
 */
public static String addCommand(List<String> cmd, boolean isExecutable) throws IOException {
    StringBuffer command = new StringBuffer();
    for (String s : cmd) {
        command.append('\'');
        if (isExecutable) {
            // the executable name needs to be expressed as a shell path for the
            // shell to find it.
            command.append(FileUtil.makeShellPath(new File(s)));
            isExecutable = false;
        } else {
            command.append(s);
        }
        command.append('\'');
        command.append(" ");
    }
    return command.toString();
}

---