Example usage for org.apache.hadoop.fs ChecksumFileSystem getChecksumFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs ChecksumFileSystem getChecksumFile.

Prototype

public Path getChecksumFile(Path file) 

Source Link

Document

Return the name of the checksum file associated with a file.

Usage

From source file:org.springframework.data.hadoop.fs.FsShell.java

License:Apache License

private void copyToLocal(final FileSystem srcFS, final Path src, final File dst, final boolean copyCrc)
        throws IOException {

    final String COPYTOLOCAL_PREFIX = "_copyToLocal_";

    /* Keep the structure similar to ChecksumFileSystem.copyToLocal(). 
    * Ideal these two should just invoke FileUtil.copy() and not repeat
    * recursion here. Of course, copy() should support two more options :
    * copyCrc and useTmpFile (may be useTmpFile need not be an option).
    *///from w ww.  j  av a  2  s.  c o  m
    if (!srcFS.getFileStatus(src).isDir()) {
        if (dst.exists()) {
            // match the error message in FileUtil.checkDest():
            throw new IOException("Target " + dst + " already exists");
        }

        // use absolute name so that tmp file is always created under dest dir
        File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(), COPYTOLOCAL_PREFIX, true);
        if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) {
            throw new IOException("Failed to copy " + src + " to " + dst);
        }

        if (!tmp.renameTo(dst)) {
            throw new IOException(
                    "Failed to rename tmp file " + tmp + " to local destination \"" + dst + "\".");
        }

        if (copyCrc) {
            if (!(srcFS instanceof ChecksumFileSystem)) {
                throw new IOException("Source file system does not have crc files");
            }

            ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
            File dstcs = FileSystem.getLocal(srcFS.getConf())
                    .pathToFile(csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
            copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false);
        }
    } else {
        // once FileUtil.copy() supports tmp file, we don't need to mkdirs().
        dst.mkdirs();
        for (FileStatus path : srcFS.listStatus(src)) {
            copyToLocal(srcFS, path.getPath(), new File(dst, path.getPath().getName()), copyCrc);
        }
    }
}

---