Example usage for org.apache.hadoop.fs FileChecksum equals

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileChecksum equals.

Prototype

@Override
public boolean equals(Object other) 

Source Link

Document

Return true if both the algorithms and the values are the same.

Usage

From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java

License:Apache License

/**
 * Utility to compare checksums for the paths specified.
 *
 * If checksums's can't be retrieved, it doesn't fail the test
 * Only time the comparison would fail is when checksums are
 * available and they don't match/*from w w w .  ja va  2 s.  c  o  m*/
 *                                  
 * @param sourceFS FileSystem for the source path.
 * @param source The source path.
 * @param targetFS FileSystem for the target path.
 * @param target The target path.
 * @return If either checksum couldn't be retrieved, the function returns
 * false. If checksums are retrieved, the function returns true if they match,
 * and false otherwise.
 * @throws IOException if there's an exception while retrieving checksums.
 */
public static boolean checksumsAreEqual(FileSystem sourceFS, Path source, FileSystem targetFS, Path target)
        throws IOException {
    try {
        FileChecksum sourceChecksum = sourceFS.getFileChecksum(source);
        if (sourceChecksum == null) {
            LOG.warn("Checksum for " + source + " is null. Checksum match skipped.");
            return true;
        }

        FileChecksum targetChecksum = targetFS.getFileChecksum(target);
        if (targetChecksum == null) {
            LOG.warn("Checksum for " + source + " is null. Checksum match skipped.");
            return true;
        }

        return (sourceChecksum.equals(targetChecksum));

    } catch (IOException e) {
        LOG.error("Unable to retrieve checksum for " + source + " or " + target, e);
        return true;
    }
}

---

From source file:com.pinterest.hdfsbackup.distcp.DistCp.java

License:Apache License

/**
 * Check whether the contents of src and dst are the same.
 *
 * Return false if dstpath does not exist
 *
 * If the files have different sizes, return false.
 *
 * If the files have the same sizes, the file checksums will be compared.
 *
 * When file checksum is not supported in any of file systems,
 * two files are considered as the same if they have the same size.
 *///ww  w  .j a  va  2  s  .c o m
static private boolean sameFile(FileSystem srcfs, FileStatus srcstatus, FileSystem dstfs, Path dstpath)
        throws IOException {
    FileStatus dststatus;
    try {
        dststatus = dstfs.getFileStatus(dstpath);
    } catch (FileNotFoundException fnfe) {
        return false;
    }

    //same length?
    if (srcstatus.getLen() != dststatus.getLen()) {
        return false;
    }

    //compare checksums
    try {
        final FileChecksum srccs = srcfs.getFileChecksum(srcstatus.getPath());
        final FileChecksum dstcs = dstfs.getFileChecksum(dststatus.getPath());
        //return true if checksum is not supported
        //(i.e. some of the checksums is null)
        return srccs == null || dstcs == null || srccs.equals(dstcs);
    } catch (FileNotFoundException fnfe) {
        return false;
    }
}

---

From source file:com.scaleunlimited.cascading.DistCp.java

License:Apache License

/**
 * Check whether the contents of src and dst are the same.
 * // w w w  . ja  v  a2  s. c o  m
 * Return false if dstpath does not exist
 * 
 * If the files have different sizes, return false.
 * 
 * If the files have the same sizes, the file checksums will be compared.
 * 
 * When file checksum is not supported in any of file systems,
 * two files are considered as the same if they have the same size.
 */
static private boolean sameFile(FileSystem srcfs, FileStatus srcstatus, FileSystem dstfs, Path dstpath)
        throws IOException {
    FileStatus dststatus;
    try {
        dststatus = dstfs.getFileStatus(dstpath);
    } catch (FileNotFoundException fnfe) {
        return false;
    }

    //same length?
    if (srcstatus.getLen() != dststatus.getLen()) {
        return false;
    }

    //get src checksum
    final FileChecksum srccs;
    try {
        srccs = srcfs.getFileChecksum(srcstatus.getPath());
    } catch (FileNotFoundException fnfe) {
        /*
         * Two possible cases:
         * (1) src existed once but was deleted between the time period that
         *     srcstatus was obtained and the try block above.
         * (2) srcfs does not support file checksum and (incorrectly) throws
         *     FNFE, e.g. some previous versions of HftpFileSystem.
         * For case (1), it is okay to return true since src was already deleted.
         * For case (2), true should be returned.  
         */
        return true;
    }

    //compare checksums
    try {
        final FileChecksum dstcs = dstfs.getFileChecksum(dststatus.getPath());
        //return true if checksum is not supported
        //(i.e. some of the checksums is null)
        return srccs == null || dstcs == null || srccs.equals(dstcs);
    } catch (FileNotFoundException fnfe) {
        return false;
    }
}

---

From source file:fr.ens.biologie.genomique.eoulsan.modules.mgmt.hadoop.DistCp.java

License:LGPL

/**
 * Check whether the contents of src and dst are the same. Return false if
 * dstpath does not exist If the files have different sizes, return false. If
 * the files have the same sizes, the file checksums will be compared. When
 * file checksum is not supported in any of file systems, two files are
 * considered as the same if they have the same size.
 *//*from   w  w w  .j av a 2 s.  c o m*/
static private boolean sameFile(final FileSystem srcfs, final FileStatus srcstatus, final FileSystem dstfs,
        final Path dstpath) throws IOException {
    FileStatus dststatus;
    try {
        dststatus = dstfs.getFileStatus(dstpath);
    } catch (FileNotFoundException fnfe) {
        return false;
    }

    // same length?
    if (srcstatus.getLen() != dststatus.getLen()) {
        return false;
    }

    // get src checksum
    final FileChecksum srccs;
    try {
        srccs = srcfs.getFileChecksum(srcstatus.getPath());
    } catch (FileNotFoundException fnfe) {
        /*
         * Two possible cases: (1) src existed once but was deleted between the
         * time period that srcstatus was obtained and the try block above. (2)
         * srcfs does not support file checksum and (incorrectly) throws FNFE,
         * e.g. some previous versions of HftpFileSystem. For case (1), it is okay
         * to return true since src was already deleted. For case (2), true should
         * be returned.
         */
        return true;
    }

    // compare checksums
    try {
        final FileChecksum dstcs = dstfs.getFileChecksum(dststatus.getPath());
        // return true if checksum is not supported
        // (i.e. some of the checksums is null)
        return srccs == null || dstcs == null || srccs.equals(dstcs);
    } catch (FileNotFoundException fnfe) {
        return false;
    }
}

---

From source file:org.jd.copier.mapred.DistCp.java

License:Apache License

/**
 * Check whether the contents of src and dst are the same.
 * /*from   w w w.j a va2 s. co m*/
 * Return false if dstpath does not exist
 * 
 * If the files have different sizes, return false.
 * 
 * If the files have the same sizes, the file checksums will be compared.
 * 
 * When file checksum is not supported in any of file systems,
 * two files are considered as the same if they have the same size.
 */
static private boolean sameFile(FileSystem srcfs, FileStatus srcstatus, FileSystem dstfs, Path dstpath,
        boolean skipCRCCheck) throws IOException {
    FileStatus dststatus;
    try {
        dststatus = dstfs.getFileStatus(dstpath);
    } catch (FileNotFoundException fnfe) {
        return false;
    }

    //same length?
    if (srcstatus.getLen() != dststatus.getLen()) {
        return false;
    }

    if (skipCRCCheck) {
        LOG.debug("Skipping CRC Check");
        return true;
    }

    //get src checksum
    final FileChecksum srccs;
    try {
        srccs = srcfs.getFileChecksum(srcstatus.getPath());
    } catch (FileNotFoundException fnfe) {
        /*
         * Two possible cases:
         * (1) src existed once but was deleted between the time period that
         *     srcstatus was obtained and the try block above.
         * (2) srcfs does not support file checksum and (incorrectly) throws
         *     FNFE, e.g. some previous versions of HftpFileSystem.
         * For case (1), it is okay to return true since src was already deleted.
         * For case (2), true should be returned.  
         */
        return true;
    }

    //compare checksums
    try {
        final FileChecksum dstcs = dstfs.getFileChecksum(dststatus.getPath());
        //return true if checksum is not supported
        //(i.e. some of the checksums is null)
        return srccs == null || dstcs == null || srccs.equals(dstcs);
    } catch (FileNotFoundException fnfe) {
        return false;
    }
}

---