Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.jxtadoop.fs; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.text.DecimalFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.*; import java.util.zip.GZIPInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.jxtadoop.conf.Configuration; import org.apache.jxtadoop.conf.Configured; import org.apache.jxtadoop.fs.shell.CommandFormat; import org.apache.jxtadoop.fs.shell.Count; import org.apache.jxtadoop.hdfs.DFSClient; import org.apache.jxtadoop.io.DataInputBuffer; import org.apache.jxtadoop.io.DataOutputBuffer; import org.apache.jxtadoop.io.IOUtils; import org.apache.jxtadoop.io.SequenceFile; import org.apache.jxtadoop.io.Writable; import org.apache.jxtadoop.io.WritableComparable; import org.apache.jxtadoop.ipc.RPC; import org.apache.jxtadoop.ipc.RemoteException; import org.apache.jxtadoop.util.ReflectionUtils; import org.apache.jxtadoop.util.Tool; import org.apache.jxtadoop.util.ToolRunner; import org.apache.jxtadoop.util.StringUtils; /** Provide command line access to a FileSystem. */ @SuppressWarnings({ "unused", "deprecation", "rawtypes" }) public class FsShell extends Configured implements Tool { static { Configuration.addDefaultResource("hdfs-default.xml"); Configuration.addDefaultResource("hdfs-site.xml"); Configuration.addDefaultResource("hdfs-p2p.xml"); } public static final Log LOG = LogFactory.getLog(FsShell.class); private DFSClient dfs; protected FileSystem fs; private Trash trash; public static final SimpleDateFormat dateForm = new SimpleDateFormat("yyyy-MM-dd HH:mm"); protected static final SimpleDateFormat modifFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); static final int BORDER = 2; static { modifFmt.setTimeZone(TimeZone.getTimeZone("UTC")); } static final String SETREP_SHORT_USAGE = "-setrep [-R] [-w] <rep> <path/file>"; static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] <src> <localdst>"; static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace("-get", "-copyToLocal"); static final String TAIL_USAGE = "-tail [-f] <file>"; /** */ public FsShell() { // this(null); this(new Configuration()); } public FsShell(Configuration conf) { super(conf); fs = null; trash = null; try { dfs = new DFSClient(conf, "FsShell"); } catch (IOException e) { LOG.fatal("Failed to initialize DFS client; Aborting"); // e.printStackTrace(); LOG.debug(e.getMessage()); } } protected void init() throws IOException { if (dfs == null) return; dfs.init(); getConf().set("fs.default.name", dfs.getDfsClientPeer().getNameNodePeerId().toString().replaceAll("urn:jxta:cbid-", "")); getConf().setQuietMode(true); if (this.fs == null) { this.fs = FileSystem.get(getConf()); } LOG.debug("fs : " + fs.getName()); if (this.trash == null) { this.trash = new Trash(getConf()); } } /** * Copies from stdin to the indicated file. */ private void copyFromStdin(Path dst, FileSystem dstFs) throws IOException { if (dstFs.isDirectory(dst)) { throw new IOException("When source is stdin, destination must be a file."); } if (dstFs.exists(dst)) { throw new IOException("Target " + dst.toString() + " already exists."); } FSDataOutputStream out = dstFs.create(dst); try { IOUtils.copyBytes(System.in, out, getConf(), false); } finally { out.close(); } } /** * Print from src to stdout. */ private void printToStdout(InputStream in) throws IOException { try { IOUtils.copyBytes(in, System.out, getConf(), false); } finally { in.close(); } } /** * Add local files to the indicated FileSystem name. src is kept. */ void copyFromLocal(Path[] srcs, String dstf) throws IOException { Path dstPath = new Path(dstf); FileSystem dstFs = dstPath.getFileSystem(getConf()); if (srcs.length == 1 && srcs[0].toString().equals("-")) copyFromStdin(dstPath, dstFs); else dstFs.copyFromLocalFile(false, false, srcs, dstPath); } /** * Add local files to the indicated FileSystem name. src is removed. */ void moveFromLocal(Path[] srcs, String dstf) throws IOException { Path dstPath = new Path(dstf); FileSystem dstFs = dstPath.getFileSystem(getConf()); dstFs.moveFromLocalFile(srcs, dstPath); } /** * Add a local file to the indicated FileSystem name. src is removed. */ void moveFromLocal(Path src, String dstf) throws IOException { moveFromLocal((new Path[] { src }), dstf); } /** * Obtain the indicated files that match the file pattern <i>srcf</i> * and copy them to the local name. srcf is kept. * When copying multiple files, the destination must be a directory. * Otherwise, IOException is thrown. * @param argv: arguments * @param pos: Ignore everything before argv[pos] * @exception: IOException * @see org.apache.jxtadoop.fs.FileSystem.globStatus */ void copyToLocal(String[] argv, int pos) throws IOException { CommandFormat cf = new CommandFormat("copyToLocal", 2, 2, "crc", "ignoreCrc"); String srcstr = null; String dststr = null; try { List<String> parameters = cf.parse(argv, pos); srcstr = parameters.get(0); dststr = parameters.get(1); } catch (IllegalArgumentException iae) { System.err.println("Usage: java FsShell " + GET_SHORT_USAGE); throw iae; } boolean copyCrc = cf.getOpt("crc"); final boolean verifyChecksum = !cf.getOpt("ignoreCrc"); if (dststr.equals("-")) { if (copyCrc) { System.err.println("-crc option is not valid when destination is stdout."); } cat(srcstr, verifyChecksum); } else { File dst = new File(dststr); Path srcpath = new Path(srcstr); FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum); if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) { System.err.println("-crc option is not valid when source file system " + "does not have crc files. Automatically turn the option off."); copyCrc = false; } FileStatus[] srcs = srcFS.globStatus(srcpath); boolean dstIsDir = dst.isDirectory(); if (srcs.length > 1 && !dstIsDir) { throw new IOException("When copying multiple files, " + "destination should be a directory."); } for (FileStatus status : srcs) { Path p = status.getPath(); File f = dstIsDir ? new File(dst, p.getName()) : dst; copyToLocal(srcFS, p, f, copyCrc); } } } /** * Return the {@link FileSystem} specified by src and the conf. * It the {@link FileSystem} supports checksum, set verifyChecksum. */ private FileSystem getSrcFileSystem(Path src, boolean verifyChecksum) throws IOException { FileSystem srcFs = src.getFileSystem(getConf()); srcFs.setVerifyChecksum(verifyChecksum); return srcFs; } /** * The prefix for the tmp file used in copyToLocal. * It must be at least three characters long, required by * {@link java.io.File#createTempFile(String, String, File)}. */ static final String COPYTOLOCAL_PREFIX = "_copyToLocal_"; /** * Copy a source file from a given file system to local destination. * @param srcFS source file system * @param src source path * @param dst destination * @param copyCrc copy CRC files? * @exception IOException If some IO failed */ private void copyToLocal(final FileSystem srcFS, final Path src, final File dst, final boolean copyCrc) throws IOException { /* Keep the structure similar to ChecksumFileSystem.copyToLocal(). * Ideal these two should just invoke FileUtil.copy() and not repeat * recursion here. Of course, copy() should support two more options : * copyCrc and useTmpFile (may be useTmpFile need not be an option). */ if (!srcFS.getFileStatus(src).isDir()) { if (dst.exists()) { // match the error message in FileUtil.checkDest(): throw new IOException("Target " + dst + " already exists"); } // use absolute name so that tmp file is always created under dest dir File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(), COPYTOLOCAL_PREFIX, true); if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) { throw new IOException("Failed to copy " + src + " to " + dst); } if (!tmp.renameTo(dst)) { throw new IOException( "Failed to rename tmp file " + tmp + " to local destination \"" + dst + "\"."); } if (copyCrc) { if (!(srcFS instanceof ChecksumFileSystem)) { throw new IOException("Source file system does not have crc files"); } ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS; File dstcs = FileSystem.getLocal(srcFS.getConf()) .pathToFile(csfs.getChecksumFile(new Path(dst.getCanonicalPath()))); copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false); } } else { // once FileUtil.copy() supports tmp file, we don't need to mkdirs(). dst.mkdirs(); for (FileStatus path : srcFS.listStatus(src)) { copyToLocal(srcFS, path.getPath(), new File(dst, path.getPath().getName()), copyCrc); } } } /** * Get all the files in the directories that match the source file * pattern and merge and sort them to only one file on local fs * srcf is kept. * @param srcf: a file pattern specifying source files * @param dstf: a destination local file/directory * @exception: IOException * @see org.apache.jxtadoop.fs.FileSystem.globStatus */ void copyMergeToLocal(String srcf, Path dst) throws IOException { copyMergeToLocal(srcf, dst, false); } /** * Get all the files in the directories that match the source file pattern * and merge and sort them to only one file on local fs * srcf is kept. * * Also adds a string between the files (useful for adding \n * to a text file) * @param srcf: a file pattern specifying source files * @param dstf: a destination local file/directory * @param endline: if an end of line character is added to a text file * @exception: IOException * @see org.apache.jxtadoop.fs.FileSystem.globStatus */ void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for (int i = 0; i < srcs.length; i++) { if (endline) { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), "\n"); } else { FileUtil.copyMerge(srcFs, srcs[i], FileSystem.getLocal(getConf()), dst, false, getConf(), null); } } } /** * Obtain the indicated file and copy to the local name. * srcf is removed. */ void moveToLocal(String srcf, Path dst) throws IOException { System.err.println("Option '-moveToLocal' is not implemented yet."); } /** * Fetch all files that match the file pattern <i>srcf</i> and display * their content on stdout. * @param srcf: a file pattern specifying source files * @exception: IOException * @see org.apache.jxtadoop.fs.FileSystem.globStatus */ void cat(String src, boolean verifyChecksum) throws IOException { //cat behavior in Linux // [~/1207]$ ls ?.txt // x.txt z.txt // [~/1207]$ cat x.txt y.txt z.txt // xxx // cat: y.txt: No such file or directory // zzz Path srcPattern = new Path(src); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { if (srcFs.getFileStatus(p).isDir()) { throw new IOException("Source must be a file."); } printToStdout(srcFs.open(p)); } }.globAndProcess(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum)); } private class TextRecordInputStream extends InputStream { SequenceFile.Reader r; WritableComparable key; Writable val; DataInputBuffer inbuf; DataOutputBuffer outbuf; public TextRecordInputStream(FileStatus f) throws IOException { r = new SequenceFile.Reader(fs, f.getPath(), getConf()); key = ReflectionUtils.newInstance(r.getKeyClass().asSubclass(WritableComparable.class), getConf()); val = ReflectionUtils.newInstance(r.getValueClass().asSubclass(Writable.class), getConf()); inbuf = new DataInputBuffer(); outbuf = new DataOutputBuffer(); } public int read() throws IOException { int ret; if (null == inbuf || -1 == (ret = inbuf.read())) { if (!r.next(key, val)) { return -1; } byte[] tmp = key.toString().getBytes(); outbuf.write(tmp, 0, tmp.length); outbuf.write('\t'); tmp = val.toString().getBytes(); outbuf.write(tmp, 0, tmp.length); outbuf.write('\n'); inbuf.reset(outbuf.getData(), outbuf.getLength()); outbuf.reset(); ret = inbuf.read(); } return ret; } } private InputStream forMagic(Path p, FileSystem srcFs) throws IOException { FSDataInputStream i = srcFs.open(p); switch (i.readShort()) { case 0x1f8b: // RFC 1952 i.seek(0); return new GZIPInputStream(i); case 0x5345: // 'S' 'E' if (i.readByte() == 'Q') { i.close(); return new TextRecordInputStream(srcFs.getFileStatus(p)); } break; } i.seek(0); return i; } void text(String srcf) throws IOException { Path srcPattern = new Path(srcf); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { if (srcFs.isDirectory(p)) { throw new IOException("Source must be a file."); } printToStdout(forMagic(p, srcFs)); } }.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf())); } /** * Parse the incoming command string * @param cmd * @param pos ignore anything before this pos in cmd * @throws IOException */ private void setReplication(String[] cmd, int pos) throws IOException { CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w"); String dst = null; short rep = 0; try { List<String> parameters = c.parse(cmd, pos); rep = Short.parseShort(parameters.get(0)); dst = parameters.get(1); } catch (NumberFormatException nfe) { System.err.println("Illegal replication, a positive integer expected"); throw nfe; } catch (IllegalArgumentException iae) { System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE); throw iae; } if (rep < 1) { System.err.println("Cannot set replication to: " + rep); throw new IllegalArgumentException("replication must be >= 1"); } List<Path> waitList = c.getOpt("w") ? new ArrayList<Path>() : null; setReplication(rep, dst, c.getOpt("R"), waitList); if (waitList != null) { waitForReplication(waitList, rep); } } /** * Wait for all files in waitList to have replication number equal to rep. * @param waitList The files are waited for. * @param rep The new replication number. * @throws IOException IOException */ void waitForReplication(List<Path> waitList, int rep) throws IOException { for (Path f : waitList) { System.out.print("Waiting for " + f + " ..."); System.out.flush(); boolean printWarning = false; FileStatus status = fs.getFileStatus(f); long len = status.getLen(); for (boolean done = false; !done;) { BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len); int i = 0; for (; i < locations.length && locations[i].getHosts().length == rep; i++) if (!printWarning && locations[i].getHosts().length > rep) { System.out.println("\nWARNING: the waiting time may be long for " + "DECREASING the number of replication."); printWarning = true; } done = i == locations.length; if (!done) { System.out.print("."); System.out.flush(); try { Thread.sleep(10000); } catch (InterruptedException e) { } } } System.out.println(" done"); } } /** * Set the replication for files that match file pattern <i>srcf</i> * if it's a directory and recursive is true, * set replication for all the subdirs and those files too. * @param newRep new replication factor * @param srcf a file pattern specifying source files * @param recursive if need to set replication factor for files in subdirs * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void setReplication(short newRep, String srcf, boolean recursive, List<Path> waitingList) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for (int i = 0; i < srcs.length; i++) { setReplication(newRep, srcFs, srcs[i], recursive, waitingList); } } private void setReplication(short newRep, FileSystem srcFs, Path src, boolean recursive, List<Path> waitingList) throws IOException { if (!srcFs.getFileStatus(src).isDir()) { setFileReplication(src, srcFs, newRep, waitingList); return; } FileStatus items[] = srcFs.listStatus(src); if (items == null) { throw new IOException("Could not get listing for " + src); } else { for (int i = 0; i < items.length; i++) { if (!items[i].isDir()) { setFileReplication(items[i].getPath(), srcFs, newRep, waitingList); } else if (recursive) { setReplication(newRep, srcFs, items[i].getPath(), recursive, waitingList); } } } } /** * Actually set the replication for this file * If it fails either throw IOException or print an error msg * @param file: a file/directory * @param newRep: new replication factor * @throws IOException */ private void setFileReplication(Path file, FileSystem srcFs, short newRep, List<Path> waitList) throws IOException { if (srcFs.setReplication(file, newRep)) { if (waitList != null) { waitList.add(file); } System.out.println("Replication " + newRep + " set: " + file); } else { System.err.println("Could not set replication for: " + file); } } /** * Get a listing of all files in that match the file pattern <i>srcf</i>. * @param srcf a file pattern specifying source files * @param recursive if need to list files in subdirs * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ private int ls(String srcf, boolean recursive) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(this.getConf()); FileStatus[] srcs = srcFs.globStatus(srcPath); if (srcs == null || srcs.length == 0) { throw new FileNotFoundException("Cannot access " + srcf + ": No such file or directory."); } boolean printHeader = (srcs.length == 1) ? true : false; int numOfErrors = 0; for (int i = 0; i < srcs.length; i++) { numOfErrors += ls(srcs[i], srcFs, recursive, printHeader); } return numOfErrors == 0 ? 0 : -1; } /* list all files under the directory <i>src</i> * ideally we should provide "-l" option, that lists like "ls -l". */ private int ls(FileStatus src, FileSystem srcFs, boolean recursive, boolean printHeader) throws IOException { final String cmd = recursive ? "lsr" : "ls"; final FileStatus[] items = shellListStatus(cmd, srcFs, src); if (items == null) { return 1; } else { int numOfErrors = 0; if (!recursive && printHeader) { if (items.length != 0) { System.out.println("Found " + items.length + " items"); } } int maxReplication = 3, maxLen = 10, maxOwner = 0, maxGroup = 0; for (int i = 0; i < items.length; i++) { FileStatus stat = items[i]; int replication = String.valueOf(stat.getReplication()).length(); int len = String.valueOf(stat.getLen()).length(); int owner = String.valueOf(stat.getOwner()).length(); int group = String.valueOf(stat.getGroup()).length(); if (replication > maxReplication) maxReplication = replication; if (len > maxLen) maxLen = len; if (owner > maxOwner) maxOwner = owner; if (group > maxGroup) maxGroup = group; } for (int i = 0; i < items.length; i++) { FileStatus stat = items[i]; Path cur = stat.getPath(); String mdate = dateForm.format(new Date(stat.getModificationTime())); System.out.print((stat.isDir() ? "d" : "-") + stat.getPermission() + " "); System.out.printf("%" + maxReplication + "s ", (!stat.isDir() ? stat.getReplication() : "-")); if (maxOwner > 0) System.out.printf("%-" + maxOwner + "s ", stat.getOwner()); if (maxGroup > 0) System.out.printf("%-" + maxGroup + "s ", stat.getGroup()); System.out.printf("%" + maxLen + "d ", stat.getLen()); System.out.print(mdate + " "); System.out.println(cur.toUri().getPath()); if (recursive && stat.isDir()) { numOfErrors += ls(stat, srcFs, recursive, printHeader); } } return numOfErrors; } } /** * Show the size of all files that match the file pattern <i>src</i> * @param src a file pattern specifying source files * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void du(String src) throws IOException { Path srcPath = new Path(src); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path[] pathItems = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); FileStatus items[] = srcFs.listStatus(pathItems); if ((items == null) || ((items.length == 0) && (!srcFs.exists(srcPath)))) { throw new FileNotFoundException("Cannot access " + src + ": No such file or directory."); } else { System.out.println("Found " + items.length + " items"); int maxLength = 10; long length[] = new long[items.length]; for (int i = 0; i < items.length; i++) { length[i] = items[i].isDir() ? srcFs.getContentSummary(items[i].getPath()).getLength() : items[i].getLen(); int len = String.valueOf(length[i]).length(); if (len > maxLength) maxLength = len; } for (int i = 0; i < items.length; i++) { System.out.printf("%-" + (maxLength + BORDER) + "d", length[i]); System.out.println(items[i].getPath()); } } } /** * Show the summary disk usage of each dir/file * that matches the file pattern <i>src</i> * @param src a file pattern specifying source files * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void dus(String src) throws IOException { Path srcPath = new Path(src); FileSystem srcFs = srcPath.getFileSystem(getConf()); FileStatus status[] = srcFs.globStatus(new Path(src)); if (status == null || status.length == 0) { throw new FileNotFoundException("Cannot access " + src + ": No such file or directory."); } for (int i = 0; i < status.length; i++) { long totalSize = srcFs.getContentSummary(status[i].getPath()).getLength(); String pathStr = status[i].getPath().toString(); System.out.println(("".equals(pathStr) ? "." : pathStr) + "\t" + totalSize); } } /** * Create the given dir */ void mkdir(String src) throws IOException { Path f = new Path(src); FileSystem srcFs = f.getFileSystem(getConf()); FileStatus fstatus = null; try { fstatus = srcFs.getFileStatus(f); if (fstatus.isDir()) { throw new IOException("cannot create directory " + src + ": File exists"); } else { throw new IOException(src + " exists but " + "is not a directory"); } } catch (FileNotFoundException e) { if (!srcFs.mkdirs(f)) { throw new IOException("failed to create " + src); } } } /** * (Re)create zero-length file at the specified path. * This will be replaced by a more UNIX-like touch when files may be * modified. */ void touchz(String src) throws IOException { Path f = new Path(src); FileSystem srcFs = f.getFileSystem(getConf()); FileStatus st; if (srcFs.exists(f)) { st = srcFs.getFileStatus(f); if (st.isDir()) { // TODO: handle this throw new IOException(src + " is a directory"); } else if (st.getLen() != 0) throw new IOException(src + " must be a zero-length file"); } FSDataOutputStream out = srcFs.create(f); out.close(); } /** * Compute the checksum of the file. * This will get enhanced in the future with multiple files handling. * Can be used for backup and restore purposes. */ void checksum(String src) throws IOException { LOG.debug("Computing checksum for file " + src); Path f = new Path(src); FileSystem fs = f.getFileSystem(getConf()); FileStatus fst; if (fs.exists(f)) { fst = fs.getFileStatus(f); if (fst.isDir()) throw new IOException(src + " is a direcory. Checksum operation not supported."); else { FileChecksum checksum = fs.getFileChecksum(f); String checksumString = checksum.toString(); int checksumLength = checksumString.length(); System.out.printf("%s\t%s\n", src, checksumString.substring(checksumLength - 33, checksumLength)); } } else { throw new IOException(src + " file does not exist"); } } /** * Check file types. */ int test(String argv[], int i) throws IOException { if (!argv[i].startsWith("-") || argv[i].length() > 2) throw new IOException("Not a flag: " + argv[i]); char flag = argv[i].toCharArray()[1]; Path f = new Path(argv[++i]); FileSystem srcFs = f.getFileSystem(getConf()); switch (flag) { case 'e': return srcFs.exists(f) ? 0 : 1; case 'z': return srcFs.getFileStatus(f).getLen() == 0 ? 0 : 1; case 'd': return srcFs.getFileStatus(f).isDir() ? 0 : 1; default: throw new IOException("Unknown flag: " + flag); } } /** * Print statistics about path in specified format. * Format sequences: * %b: Size of file in blocks * %n: Filename * %o: Block size * %r: replication * %y: UTC date as "yyyy-MM-dd HH:mm:ss" * %Y: Milliseconds since January 1, 1970 UTC */ void stat(char[] fmt, String src) throws IOException { Path srcPath = new Path(src); FileSystem srcFs = srcPath.getFileSystem(getConf()); FileStatus glob[] = srcFs.globStatus(srcPath); if (null == glob) throw new IOException("cannot stat `" + src + "': No such file or directory"); for (FileStatus f : glob) { StringBuilder buf = new StringBuilder(); for (int i = 0; i < fmt.length; ++i) { if (fmt[i] != '%') { buf.append(fmt[i]); } else { if (i + 1 == fmt.length) break; switch (fmt[++i]) { case 'b': buf.append(f.getLen()); break; case 'F': buf.append(f.isDir() ? "directory" : "regular file"); break; case 'n': buf.append(f.getPath().getName()); break; case 'o': buf.append(f.getBlockSize()); break; case 'r': buf.append(f.getReplication()); break; case 'y': buf.append(modifFmt.format(new Date(f.getModificationTime()))); break; case 'Y': buf.append(f.getModificationTime()); break; default: buf.append(fmt[i]); break; } } } System.out.println(buf.toString()); } } /** * Move files that match the file pattern <i>srcf</i> * to a destination file. * When moving mutiple files, the destination must be a directory. * Otherwise, IOException is thrown. * @param srcf a file pattern specifying source files * @param dstf a destination local file/directory * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void rename(String srcf, String dstf) throws IOException { Path srcPath = new Path(srcf); Path dstPath = new Path(dstf); FileSystem srcFs = srcPath.getFileSystem(getConf()); FileSystem dstFs = dstPath.getFileSystem(getConf()); URI srcURI = srcFs.getUri(); URI dstURI = dstFs.getUri(); if (srcURI.compareTo(dstURI) != 0) { throw new IOException("src and destination filesystems do not match."); } Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); Path dst = new Path(dstf); if (srcs.length > 1 && !srcFs.isDirectory(dst)) { throw new IOException("When moving multiple files, " + "destination should be a directory."); } for (int i = 0; i < srcs.length; i++) { if (!srcFs.rename(srcs[i], dst)) { FileStatus srcFstatus = null; FileStatus dstFstatus = null; try { srcFstatus = srcFs.getFileStatus(srcs[i]); } catch (FileNotFoundException e) { throw new FileNotFoundException(srcs[i] + ": No such file or directory"); } try { dstFstatus = dstFs.getFileStatus(dst); } catch (IOException e) { } if ((srcFstatus != null) && (dstFstatus != null)) { if (srcFstatus.isDir() && !dstFstatus.isDir()) { throw new IOException( "cannot overwrite non directory " + dst + " with directory " + srcs[i]); } } throw new IOException("Failed to rename " + srcs[i] + " to " + dst); } } } /** * Move/rename file(s) to a destination file. Multiple source * files can be specified. The destination is the last element of * the argvp[] array. * If multiple source files are specified, then the destination * must be a directory. Otherwise, IOException is thrown. * @exception: IOException */ private int rename(String argv[], Configuration conf) throws IOException { int i = 0; int exitCode = 0; String cmd = argv[i++]; String dest = argv[argv.length - 1]; // // If the user has specified multiple source files, then // the destination has to be a directory // if (argv.length > 3) { Path dst = new Path(dest); FileSystem dstFs = dst.getFileSystem(getConf()); if (!dstFs.isDirectory(dst)) { throw new IOException( "When moving multiple files, " + "destination " + dest + " should be a directory."); } } // // for each source file, issue the rename // for (; i < argv.length - 1; i++) { try { // // issue the rename to the fs // rename(argv[i], dest); } catch (RemoteException e) { // // This is a error returned by hadoop server. Print // out the first line of the error mesage. // exitCode = -1; try { String[] content; content = e.getLocalizedMessage().split("\n"); System.err.println(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); } } catch (IOException e) { // // IO exception encountered locally. // exitCode = -1; System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage()); } } return exitCode; } /** * Copy files that match the file pattern <i>srcf</i> * to a destination file. * When copying mutiple files, the destination must be a directory. * Otherwise, IOException is thrown. * @param srcf a file pattern specifying source files * @param dstf a destination local file/directory * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void copy(String srcf, String dstf, Configuration conf) throws IOException { Path srcPath = new Path(srcf); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path dstPath = new Path(dstf); FileSystem dstFs = dstPath.getFileSystem(getConf()); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); if (srcs.length > 1 && !dstFs.isDirectory(dstPath)) { throw new IOException("When copying multiple files, " + "destination should be a directory."); } for (int i = 0; i < srcs.length; i++) { FileUtil.copy(srcFs, srcs[i], dstFs, dstPath, false, conf); } } /** * Copy file(s) to a destination file. Multiple source * files can be specified. The destination is the last element of * the argvp[] array. * If multiple source files are specified, then the destination * must be a directory. Otherwise, IOException is thrown. * @exception: IOException */ private int copy(String argv[], Configuration conf) throws IOException { int i = 0; int exitCode = 0; String cmd = argv[i++]; String dest = argv[argv.length - 1]; // // If the user has specified multiple source files, then // the destination has to be a directory // if (argv.length > 3) { Path dst = new Path(dest); if (!fs.isDirectory(dst)) { throw new IOException( "When copying multiple files, " + "destination " + dest + " should be a directory."); } } // // for each source file, issue the copy // for (; i < argv.length - 1; i++) { try { // // issue the copy to the fs // copy(argv[i], dest, conf); } catch (RemoteException e) { // // This is a error returned by hadoop server. Print // out the first line of the error mesage. // exitCode = -1; try { String[] content; content = e.getLocalizedMessage().split("\n"); System.err.println(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); } } catch (IOException e) { // // IO exception encountered locally. // exitCode = -1; System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage()); } } return exitCode; } /** * Delete all files that match the file pattern <i>srcf</i>. * @param srcf a file pattern specifying source files * @param recursive if need to delete subdirs * @param skipTrash Should we skip the trash, if it's enabled? * @throws IOException * @see org.apache.jxtadoop.fs.FileSystem#globStatus(Path) */ void delete(String srcf, final boolean recursive, final boolean skipTrash) throws IOException { //rm behavior in Linux // [~/1207]$ ls ?.txt // x.txt z.txt // [~/1207]$ rm x.txt y.txt z.txt // rm: cannot remove `y.txt': No such file or directory Path srcPattern = new Path(srcf); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { delete(p, srcFs, recursive, skipTrash); } }.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf())); } /* delete a file */ private void delete(Path src, FileSystem srcFs, boolean recursive, boolean skipTrash) throws IOException { FileStatus fs = null; try { fs = srcFs.getFileStatus(src); } catch (FileNotFoundException fnfe) { // Have to re-throw so that console output is as expected throw new FileNotFoundException("cannot remove " + src + ": No such file or directory."); } if (fs.isDir() && !recursive) { throw new IOException("Cannot remove directory \"" + src + "\", use -rmr instead"); } if (!skipTrash) { Trash trashTmp = new Trash(srcFs, getConf()); if (trashTmp.moveToTrash(src)) { System.out.println("Moved to trash: " + src); return; } } if (srcFs.delete(src, true)) { System.out.println("Deleted " + src); } else { throw new IOException("Delete failed " + src); } } private void expunge() throws IOException { trash.expunge(); trash.checkpoint(); } /** * Returns the Trash object associated with this shell. */ public Path getCurrentTrashDir() { return trash.getCurrentTrashDir(); } /** * Parse the incoming command string * @param cmd * @param pos ignore anything before this pos in cmd * @throws IOException */ private void tail(String[] cmd, int pos) throws IOException { CommandFormat c = new CommandFormat("tail", 1, 1, "f"); String src = null; Path path = null; try { List<String> parameters = c.parse(cmd, pos); src = parameters.get(0); } catch (IllegalArgumentException iae) { System.err.println("Usage: java FsShell " + TAIL_USAGE); throw iae; } boolean foption = c.getOpt("f") ? true : false; path = new Path(src); FileSystem srcFs = path.getFileSystem(getConf()); if (srcFs.isDirectory(path)) { throw new IOException("Source must be a file."); } long fileSize = srcFs.getFileStatus(path).getLen(); long offset = (fileSize > 1024) ? fileSize - 1024 : 0; while (true) { FSDataInputStream in = srcFs.open(path); in.seek(offset); IOUtils.copyBytes(in, System.out, 1024, false); offset = in.getPos(); in.close(); if (!foption) { break; } fileSize = srcFs.getFileStatus(path).getLen(); offset = (fileSize > offset) ? offset : fileSize; try { Thread.sleep(5000); } catch (InterruptedException e) { break; } } } /** * This class runs a command on a given FileStatus. This can be used for * running various commands like chmod, chown etc. */ static abstract class CmdHandler { protected int errorCode = 0; protected boolean okToContinue = true; protected String cmdName; int getErrorCode() { return errorCode; } boolean okToContinue() { return okToContinue; } String getName() { return cmdName; } protected CmdHandler(String cmdName, FileSystem fs) { this.cmdName = cmdName; } public abstract void run(FileStatus file, FileSystem fs) throws IOException; } /** helper returns listStatus() */ private static FileStatus[] shellListStatus(String cmd, FileSystem srcFs, FileStatus src) { if (!src.isDir()) { FileStatus[] files = { src }; return files; } Path path = src.getPath(); try { FileStatus[] files = srcFs.listStatus(path); if (files == null) { System.err.println(cmd + ": could not get listing for '" + path + "'"); } return files; } catch (IOException e) { System.err.println( cmd + ": could not get get listing for '" + path + "' : " + e.getMessage().split("\n")[0]); } return null; } /** * Runs the command on a given file with the command handler. * If recursive is set, command is run recursively. */ private static int runCmdHandler(CmdHandler handler, FileStatus stat, FileSystem srcFs, boolean recursive) throws IOException { int errors = 0; handler.run(stat, srcFs); if (recursive && stat.isDir() && handler.okToContinue()) { FileStatus[] files = shellListStatus(handler.getName(), srcFs, stat); if (files == null) { return 1; } for (FileStatus file : files) { errors += runCmdHandler(handler, file, srcFs, recursive); } } return errors; } ///top level runCmdHandler int runCmdHandler(CmdHandler handler, String[] args, int startIndex, boolean recursive) throws IOException { int errors = 0; for (int i = startIndex; i < args.length; i++) { Path srcPath = new Path(args[i]); FileSystem srcFs = srcPath.getFileSystem(getConf()); Path[] paths = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath); for (Path path : paths) { try { FileStatus file = srcFs.getFileStatus(path); if (file == null) { System.err.println(handler.getName() + ": could not get status for '" + path + "'"); errors++; } else { errors += runCmdHandler(handler, file, srcFs, recursive); } } catch (IOException e) { String msg = (e.getMessage() != null ? e.getLocalizedMessage() : (e.getCause().getMessage() != null ? e.getCause().getLocalizedMessage() : "null")); System.err.println( handler.getName() + ": could not get status for '" + path + "': " + msg.split("\n")[0]); } } } return (errors > 0 || handler.getErrorCode() != 0) ? 1 : 0; } /** * Return an abbreviated English-language desc of the byte length * @deprecated Consider using {@link org.apache.jxtadoop.util.StringUtils#byteDesc} instead. */ @Deprecated public static String byteDesc(long len) { return StringUtils.byteDesc(len); } /** * @deprecated Consider using {@link org.apache.jxtadoop.util.StringUtils#limitDecimalTo2} instead. */ @Deprecated public static synchronized String limitDecimalTo2(double d) { return StringUtils.limitDecimalTo2(d); } private void printHelp(String cmd) { String summary = "hadoop fs is the command to execute fs commands. " + "The full syntax is: \n\n" + "hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t" + "[-D <property=value>] [-ls <path>] [-lsr <path>] [-du <path>]\n\t" + "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm [-skipTrash] <src>]\n\t" + "[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" + "[-moveFromLocal <localsrc> ... <dst>] [" + GET_SHORT_USAGE + "\n\t" + "[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" + "[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" + "[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" + "[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" + "[-tail [-f] <path>] [-text <path>]\n\t" + "[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" + "[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" + "[" + FsShellPermissions.CHGRP_USAGE + "]\n\t" + "[" + Count.USAGE + "]\n\t" + "[-checksum <file>]\n\t" + "[-help [cmd]]\n"; String conf = "-conf <configuration file>: Specify an application configuration file."; String D = "-D <property=value>: Use value for given property."; String fs = "-fs [local | <file system URI>]: \tSpecify the file system to use.\n" + "\t\tIf not specified, the current configuration is used, \n" + "\t\ttaken from the following, in increasing precedence: \n" + "\t\t\tcore-default.xml inside the hadoop jar file \n" + "\t\t\tcore-site.xml in $HADOOP_CONF_DIR \n" + "\t\t'local' means use the local file system as your DFS. \n" + "\t\t<file system URI> specifies a particular file system to \n" + "\t\tcontact. This argument is optional but if used must appear\n" + "\t\tappear first on the command line. Exactly one additional\n" + "\t\targument must be specified. \n"; String ls = "-ls <path>: \tList the contents that match the specified file pattern. If\n" + "\t\tpath is not specified, the contents of /user/<currentUser>\n" + "\t\twill be listed. Directory entries are of the form \n" + "\t\t\tdirName (full path) <dir> \n" + "\t\tand file entries are of the form \n" + "\t\t\tfileName(full path) <r n> size \n" + "\t\twhere n is the number of replicas specified for the file \n" + "\t\tand size is the size of the file, in bytes.\n"; String lsr = "-lsr <path>: \tRecursively list the contents that match the specified\n" + "\t\tfile pattern. Behaves very similarly to hadoop fs -ls,\n" + "\t\texcept that the data is shown for all the entries in the\n" + "\t\tsubtree.\n"; String du = "-du <path>: \tShow the amount of space, in bytes, used by the files that \n" + "\t\tmatch the specified file pattern. Equivalent to the unix\n" + "\t\tcommand \"du -sb <path>/*\" in case of a directory, \n" + "\t\tand to \"du -b <path>\" in case of a file.\n" + "\t\tThe output is in the form \n" + "\t\t\tname(full path) size (in bytes)\n"; String dus = "-dus <path>: \tShow the amount of space, in bytes, used by the files that \n" + "\t\tmatch the specified file pattern. Equivalent to the unix\n" + "\t\tcommand \"du -sb\" The output is in the form \n" + "\t\t\tname(full path) size (in bytes)\n"; String mv = "-mv <src> <dst>: Move files that match the specified file pattern <src>\n" + "\t\tto a destination <dst>. When moving multiple files, the \n" + "\t\tdestination must be a directory. \n"; String cp = "-cp <src> <dst>: Copy files that match the file pattern <src> to a \n" + "\t\tdestination. When copying multiple files, the destination\n" + "\t\tmust be a directory. \n"; String rm = "-rm [-skipTrash] <src>: \tDelete all files that match the specified file pattern.\n" + "\t\tEquivalent to the Unix command \"rm <src>\"\n" + "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" + "deletes <src>"; String rmr = "-rmr [-skipTrash] <src>: \tRemove all directories which match the specified file \n" + "\t\tpattern. Equivalent to the Unix command \"rm -rf <src>\"\n" + "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" + "deletes <src>"; String put = "-put <localsrc> ... <dst>: \tCopy files " + "from the local file system \n\t\tinto fs. \n"; String copyFromLocal = "-copyFromLocal <localsrc> ... <dst>:" + " Identical to the -put command.\n"; String moveFromLocal = "-moveFromLocal <localsrc> ... <dst>:" + " Same as -put, except that the source is\n\t\tdeleted after it's copied.\n"; String get = GET_SHORT_USAGE + ": Copy files that match the file pattern <src> \n" + "\t\tto the local name. <src> is kept. When copying mutiple, \n" + "\t\tfiles, the destination must be a directory. \n"; String getmerge = "-getmerge <src> <localdst>: Get all the files in the directories that \n" + "\t\tmatch the source file pattern and merge and sort them to only\n" + "\t\tone file on local fs. <src> is kept.\n"; String cat = "-cat <src>: \tFetch all files that match the file pattern <src> \n" + "\t\tand display their content on stdout.\n"; String text = "-text <src>: \tTakes a source file and outputs the file in text format.\n" + "\t\tThe allowed formats are zip and TextRecordInputStream.\n"; String copyToLocal = COPYTOLOCAL_SHORT_USAGE + ": Identical to the -get command.\n"; String moveToLocal = "-moveToLocal <src> <localdst>: Not implemented yet \n"; String mkdir = "-mkdir <path>: \tCreate a directory in specified location. \n"; String setrep = SETREP_SHORT_USAGE + ": Set the replication level of a file. \n" + "\t\tThe -R flag requests a recursive change of replication level \n" + "\t\tfor an entire tree.\n"; String touchz = "-touchz <path>: Write a timestamp in yyyy-MM-dd HH:mm:ss format\n" + "\t\tin a file at <path>. An error is returned if the file exists with non-zero length\n"; String test = "-test -[ezd] <path>: If file { exists, has zero length, is a directory\n" + "\t\tthen return 0, else return 1.\n"; String stat = "-stat [format] <path>: Print statistics about the file/directory at <path>\n" + "\t\tin the specified format. Format accepts filesize in blocks (%b), filename (%n),\n" + "\t\tblock size (%o), replication (%r), modification date (%y, %Y)\n"; String tail = TAIL_USAGE + ": Show the last 1KB of the file. \n" + "\t\tThe -f option shows apended data as the file grows. \n"; String chmod = FsShellPermissions.CHMOD_USAGE + "\n" + "\t\tChanges permissions of a file.\n" + "\t\tThis works similar to shell's chmod with a few exceptions.\n\n" + "\t-R\tmodifies the files recursively. This is the only option\n" + "\t\tcurrently supported.\n\n" + "\tMODE\tMode is same as mode used for chmod shell command.\n" + "\t\tOnly letters recognized are 'rwxX'. E.g. a+r,g-w,+rwx,o=r\n\n" + "\tOCTALMODE Mode specifed in 3 digits. Unlike shell command,\n" + "\t\tthis requires all three digits.\n" + "\t\tE.g. 754 is same as u=rwx,g=rx,o=r\n\n" + "\t\tIf none of 'augo' is specified, 'a' is assumed and unlike\n" + "\t\tshell command, no umask is applied.\n"; String chown = FsShellPermissions.CHOWN_USAGE + "\n" + "\t\tChanges owner and group of a file.\n" + "\t\tThis is similar to shell's chown with a few exceptions.\n\n" + "\t-R\tmodifies the files recursively. This is the only option\n" + "\t\tcurrently supported.\n\n" + "\t\tIf only owner or group is specified then only owner or\n" + "\t\tgroup is modified.\n\n" + "\t\tThe owner and group names may only cosists of digits, alphabet,\n" + "\t\tand any of '-_.@/' i.e. [-_.@/a-zA-Z0-9]. The names are case\n" + "\t\tsensitive.\n\n" + "\t\tWARNING: Avoid using '.' to separate user name and group though\n" + "\t\tLinux allows it. If user names have dots in them and you are\n" + "\t\tusing local file system, you might see surprising results since\n" + "\t\tshell command 'chown' is used for local files.\n"; String chgrp = FsShellPermissions.CHGRP_USAGE + "\n" + "\t\tThis is equivalent to -chown ... :GROUP ...\n"; String checksum = "-checksum <file>: \tDisplays the MD5 checksum of the file\n" + "\t\tin the Distributed FileSystem\n"; String help = "-help [cmd]: \tDisplays help for given command or all commands if none\n" + "\t\tis specified.\n"; if ("fs".equals(cmd)) { System.out.println(fs); } else if ("conf".equals(cmd)) { System.out.println(conf); } else if ("D".equals(cmd)) { System.out.println(D); } else if ("ls".equals(cmd)) { System.out.println(ls); } else if ("lsr".equals(cmd)) { System.out.println(lsr); } else if ("du".equals(cmd)) { System.out.println(du); } else if ("dus".equals(cmd)) { System.out.println(dus); } else if ("rm".equals(cmd)) { System.out.println(rm); } else if ("rmr".equals(cmd)) { System.out.println(rmr); } else if ("mkdir".equals(cmd)) { System.out.println(mkdir); } else if ("mv".equals(cmd)) { System.out.println(mv); } else if ("cp".equals(cmd)) { System.out.println(cp); } else if ("put".equals(cmd)) { System.out.println(put); } else if ("copyFromLocal".equals(cmd)) { System.out.println(copyFromLocal); } else if ("moveFromLocal".equals(cmd)) { System.out.println(moveFromLocal); } else if ("get".equals(cmd)) { System.out.println(get); } else if ("getmerge".equals(cmd)) { System.out.println(getmerge); } else if ("copyToLocal".equals(cmd)) { System.out.println(copyToLocal); } else if ("moveToLocal".equals(cmd)) { System.out.println(moveToLocal); } else if ("cat".equals(cmd)) { System.out.println(cat); } else if ("get".equals(cmd)) { System.out.println(get); } else if ("setrep".equals(cmd)) { System.out.println(setrep); } else if ("touchz".equals(cmd)) { System.out.println(touchz); } else if ("test".equals(cmd)) { System.out.println(test); } else if ("text".equals(cmd)) { System.out.println(text); } else if ("stat".equals(cmd)) { System.out.println(stat); } else if ("tail".equals(cmd)) { System.out.println(tail); } else if ("chmod".equals(cmd)) { System.out.println(chmod); } else if ("chown".equals(cmd)) { System.out.println(chown); } else if ("chgrp".equals(cmd)) { System.out.println(chgrp); } else if (Count.matches(cmd)) { System.out.println(Count.DESCRIPTION); } else if ("checksum".equals(cmd)) { System.out.println(checksum); } else if ("help".equals(cmd)) { System.out.println(help); } else { System.out.println(summary); System.out.println(fs); System.out.println(ls); System.out.println(lsr); System.out.println(du); System.out.println(dus); System.out.println(mv); System.out.println(cp); System.out.println(rm); System.out.println(rmr); System.out.println(put); System.out.println(copyFromLocal); System.out.println(moveFromLocal); System.out.println(get); System.out.println(getmerge); System.out.println(cat); System.out.println(copyToLocal); System.out.println(moveToLocal); System.out.println(mkdir); System.out.println(setrep); System.out.println(tail); System.out.println(touchz); System.out.println(test); System.out.println(text); System.out.println(stat); System.out.println(chmod); System.out.println(chown); System.out.println(chgrp); System.out.println(Count.DESCRIPTION); System.out.println(checksum); System.out.println(help); } } /** * Apply operation specified by 'cmd' on all parameters * starting from argv[startindex]. */ private int doall(String cmd, String argv[], int startindex) { int exitCode = 0; int i = startindex; boolean rmSkipTrash = false; // Check for -skipTrash option in rm/rmr if (("-rm".equals(cmd) || "-rmr".equals(cmd)) && "-skipTrash".equals(argv[i])) { rmSkipTrash = true; i++; } // // for each source file, issue the command // for (; i < argv.length; i++) { try { // // issue the command to the fs // if ("-cat".equals(cmd)) { cat(argv[i], true); } else if ("-mkdir".equals(cmd)) { mkdir(argv[i]); } else if ("-rm".equals(cmd)) { delete(argv[i], false, rmSkipTrash); } else if ("-rmr".equals(cmd)) { delete(argv[i], true, rmSkipTrash); } else if ("-du".equals(cmd)) { du(argv[i]); } else if ("-dus".equals(cmd)) { dus(argv[i]); } else if (Count.matches(cmd)) { new Count(argv, i, getConf()).runAll(); } else if ("-ls".equals(cmd)) { exitCode = ls(argv[i], false); } else if ("-lsr".equals(cmd)) { exitCode = ls(argv[i], true); } else if ("-touchz".equals(cmd)) { touchz(argv[i]); } else if ("-text".equals(cmd)) { text(argv[i]); } } catch (RemoteException e) { // // This is a error returned by hadoop server. Print // out the first line of the error message. // exitCode = -1; try { String[] content; content = e.getLocalizedMessage().split("\n"); System.err.println(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); } } catch (IOException e) { // // IO exception encountered locally. // exitCode = -1; String content = e.getLocalizedMessage(); if (content != null) { content = content.split("\n")[0]; } System.err.println(cmd.substring(1) + ": " + content); } } return exitCode; } /** * Displays format of commands. * */ private static void printUsage(String cmd) { String prefix = "Usage: java " + FsShell.class.getSimpleName(); if ("-fs".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-fs <local | file system URI>]"); } else if ("-conf".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-conf <configuration file>]"); } else if ("-D".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-D <[property=value>]"); } else if ("-ls".equals(cmd) || "-lsr".equals(cmd) || "-du".equals(cmd) || "-dus".equals(cmd) || "-touchz".equals(cmd) || "-mkdir".equals(cmd) || "-text".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " <path>]"); } else if (Count.matches(cmd)) { System.err.println(prefix + " [" + Count.USAGE + "]"); } else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) { System.err.println("Usage: java FsShell [" + cmd + " [-skipTrash] <src>]"); } else if ("-mv".equals(cmd) || "-cp".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " <src> <dst>]"); } else if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd) || "-moveFromLocal".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " <localsrc> ... <dst>]"); } else if ("-get".equals(cmd)) { System.err.println("Usage: java FsShell [" + GET_SHORT_USAGE + "]"); } else if ("-copyToLocal".equals(cmd)) { System.err.println("Usage: java FsShell [" + COPYTOLOCAL_SHORT_USAGE + "]"); } else if ("-moveToLocal".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " [-crc] <src> <localdst>]"); } else if ("-cat".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " <src>]"); } else if ("-setrep".equals(cmd)) { System.err.println("Usage: java FsShell [" + SETREP_SHORT_USAGE + "]"); } else if ("-test".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-test -[ezd] <path>]"); } else if ("-stat".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-stat [format] <path>]"); } else if ("-tail".equals(cmd)) { System.err.println("Usage: java FsShell [" + TAIL_USAGE + "]"); } else if ("-checksum".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-checksum <file>]"); } else { System.err.println("Usage: java FsShell"); System.err.println(" [-ls <path>]"); System.err.println(" [-lsr <path>]"); System.err.println(" [-du <path>]"); System.err.println(" [-dus <path>]"); System.err.println(" [" + Count.USAGE + "]"); System.err.println(" [-mv <src> <dst>]"); System.err.println(" [-cp <src> <dst>]"); System.err.println(" [-rm [-skipTrash] <path>]"); System.err.println(" [-rmr [-skipTrash] <path>]"); System.err.println(" [-expunge]"); System.err.println(" [-put <localsrc> ... <dst>]"); System.err.println(" [-copyFromLocal <localsrc> ... <dst>]"); System.err.println(" [-moveFromLocal <localsrc> ... <dst>]"); System.err.println(" [" + GET_SHORT_USAGE + "]"); System.err.println(" [-getmerge <src> <localdst> [addnl]]"); System.err.println(" [-cat <src>]"); System.err.println(" [-text <src>]"); System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]"); System.err.println(" [-moveToLocal [-crc] <src> <localdst>]"); System.err.println(" [-mkdir <path>]"); System.err.println(" [" + SETREP_SHORT_USAGE + "]"); System.err.println(" [-touchz <path>]"); System.err.println(" [-test -[ezd] <path>]"); System.err.println(" [-stat [format] <path>]"); System.err.println(" [" + TAIL_USAGE + "]"); System.err.println(" [" + FsShellPermissions.CHMOD_USAGE + "]"); System.err.println(" [" + FsShellPermissions.CHOWN_USAGE + "]"); System.err.println(" [" + FsShellPermissions.CHGRP_USAGE + "]"); System.err.println(" [-checksum <file>]"); System.err.println(" [-help [cmd]]"); System.err.println(); ToolRunner.printGenericCommandUsage(System.err); } } /** * run */ public int run(String argv[]) throws Exception { if (argv.length < 1) { printUsage(""); return -1; } int exitCode = -1; int i = 0; String cmd = argv[i++]; // // verify that we have enough command line parameters // if ("-put".equals(cmd) || "-test".equals(cmd) || "-copyFromLocal".equals(cmd) || "-moveFromLocal".equals(cmd)) { if (argv.length < 3) { printUsage(cmd); return exitCode; } } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd) || "-moveToLocal".equals(cmd)) { if (argv.length < 3) { printUsage(cmd); return exitCode; } } else if ("-mv".equals(cmd) || "-cp".equals(cmd)) { if (argv.length < 3) { printUsage(cmd); return exitCode; } } else if ("-rm".equals(cmd) || "-rmr".equals(cmd) || "-cat".equals(cmd) || "-mkdir".equals(cmd) || "-touchz".equals(cmd) || "-stat".equals(cmd) || "-text".equals(cmd) || "-checksum".equals(cmd)) { if (argv.length < 2) { printUsage(cmd); return exitCode; } } // initialize FsShell try { init(); } catch (RPC.VersionMismatch v) { System.err.println("Version Mismatch between client and server" + "... command aborted."); return exitCode; } catch (IOException e) { System.err.println("Bad connection to FS. command aborted."); return exitCode; } exitCode = 0; try { if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) { Path[] srcs = new Path[argv.length - 2]; for (int j = 0; i < argv.length - 1;) srcs[j++] = new Path(argv[i++]); copyFromLocal(srcs, argv[i++]); } else if ("-moveFromLocal".equals(cmd)) { Path[] srcs = new Path[argv.length - 2]; for (int j = 0; i < argv.length - 1;) srcs[j++] = new Path(argv[i++]); moveFromLocal(srcs, argv[i++]); } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) { copyToLocal(argv, i); } else if ("-getmerge".equals(cmd)) { if (argv.length > i + 2) copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++])); else copyMergeToLocal(argv[i++], new Path(argv[i++])); } else if ("-cat".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-text".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-moveToLocal".equals(cmd)) { moveToLocal(argv[i++], new Path(argv[i++])); } else if ("-setrep".equals(cmd)) { setReplication(argv, i); } else if ("-chmod".equals(cmd) || "-chown".equals(cmd) || "-chgrp".equals(cmd)) { FsShellPermissions.changePermissions(fs, cmd, argv, i, this); } else if ("-ls".equals(cmd)) { if (i < argv.length) { exitCode = doall(cmd, argv, i); } else { exitCode = ls(Path.CUR_DIR, false); } } else if ("-lsr".equals(cmd)) { if (i < argv.length) { exitCode = doall(cmd, argv, i); } else { exitCode = ls(Path.CUR_DIR, true); } } else if ("-mv".equals(cmd)) { exitCode = rename(argv, getConf()); } else if ("-cp".equals(cmd)) { exitCode = copy(argv, getConf()); } else if ("-rm".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-rmr".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-expunge".equals(cmd)) { expunge(); } else if ("-du".equals(cmd)) { if (i < argv.length) { exitCode = doall(cmd, argv, i); } else { du("."); } } else if ("-dus".equals(cmd)) { if (i < argv.length) { exitCode = doall(cmd, argv, i); } else { dus("."); } } else if (Count.matches(cmd)) { exitCode = new Count(argv, i, getConf()).runAll(); } else if ("-mkdir".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-touchz".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-test".equals(cmd)) { exitCode = test(argv, i); } else if ("-stat".equals(cmd)) { if (i + 1 < argv.length) { stat(argv[i++].toCharArray(), argv[i++]); } else { stat("%y".toCharArray(), argv[i]); } } else if ("-checksum".equals(cmd)) { checksum(argv[i++]); } else if ("-help".equals(cmd)) { if (i < argv.length) { printHelp(argv[i]); } else { printHelp(""); } } else if ("-tail".equals(cmd)) { tail(argv, i); } else { exitCode = -1; System.err.println(cmd.substring(1) + ": Unknown command"); printUsage(""); } } catch (IllegalArgumentException arge) { exitCode = -1; System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage()); printUsage(cmd); } catch (RemoteException e) { // // This is a error returned by hadoop server. Print // out the first line of the error mesage, ignore the stack trace. exitCode = -1; try { String[] content; content = e.getLocalizedMessage().split("\n"); System.err.println(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); } } catch (IOException e) { // // IO exception encountered locally. // exitCode = -1; System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage()); } catch (Exception re) { exitCode = -1; System.err.println(cmd.substring(1) + ": " + re.getLocalizedMessage()); } finally { } return exitCode; } public void close() throws IOException { if (fs != null) { fs.close(); fs = null; } if (dfs != null) { dfs.close(); dfs = null; } } /** * main() has some simple utility methods */ public static void main(String argv[]) { FsShell shell = null; int res = 0; try { LOG.debug("Starting shell with cmd : " + argv); shell = new FsShell(); res = ToolRunner.run(shell, argv); } catch (Exception e) { System.err.println("Failed to execute shell command"); System.exit(res); } finally { LOG.debug("Closing shell"); try { if (shell != null) shell.close(); } catch (IOException e) { System.err.println("Failed to closed shell connection to namenode"); System.exit(res); } } System.exit(res); } /** * Accumulate exceptions if there is any. Throw them at last. */ private abstract class DelayedExceptionThrowing { abstract void process(Path p, FileSystem srcFs) throws IOException; final void globAndProcess(Path srcPattern, FileSystem srcFs) throws IOException { List<IOException> exceptions = new ArrayList<IOException>(); for (Path p : FileUtil.stat2Paths(srcFs.globStatus(srcPattern), srcPattern)) try { process(p, srcFs); } catch (IOException ioe) { exceptions.add(ioe); } if (!exceptions.isEmpty()) if (exceptions.size() == 1) throw exceptions.get(0); else throw new IOException("Multiple IOExceptions: " + exceptions); } } }