Example usage for org.apache.hadoop.fs ChecksumFileSystem getRawFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs ChecksumFileSystem getRawFileSystem.

Prototype

@Override
public FileSystem getRawFileSystem() 

Source Link

Document

get the raw file system

Usage

From source file:org.springframework.data.hadoop.fs.FsShell.java

License:Apache License

private void copyToLocal(final FileSystem srcFS, final Path src, final File dst, final boolean copyCrc)
        throws IOException {

    final String COPYTOLOCAL_PREFIX = "_copyToLocal_";

    /* Keep the structure similar to ChecksumFileSystem.copyToLocal(). 
    * Ideal these two should just invoke FileUtil.copy() and not repeat
    * recursion here. Of course, copy() should support two more options :
    * copyCrc and useTmpFile (may be useTmpFile need not be an option).
    *///from   w  w w  .  j  av a 2s  . c  om
    if (!srcFS.getFileStatus(src).isDir()) {
        if (dst.exists()) {
            // match the error message in FileUtil.checkDest():
            throw new IOException("Target " + dst + " already exists");
        }

        // use absolute name so that tmp file is always created under dest dir
        File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(), COPYTOLOCAL_PREFIX, true);
        if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) {
            throw new IOException("Failed to copy " + src + " to " + dst);
        }

        if (!tmp.renameTo(dst)) {
            throw new IOException(
                    "Failed to rename tmp file " + tmp + " to local destination \"" + dst + "\".");
        }

        if (copyCrc) {
            if (!(srcFS instanceof ChecksumFileSystem)) {
                throw new IOException("Source file system does not have crc files");
            }

            ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
            File dstcs = FileSystem.getLocal(srcFS.getConf())
                    .pathToFile(csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
            copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false);
        }
    } else {
        // once FileUtil.copy() supports tmp file, we don't need to mkdirs().
        dst.mkdirs();
        for (FileStatus path : srcFS.listStatus(src)) {
            copyToLocal(srcFS, path.getPath(), new File(dst, path.getPath().getName()), copyCrc);
        }
    }
}

---