Example usage for org.apache.hadoop.fs ChecksumException ChecksumException

Introduction

In this page you can find the example usage for org.apache.hadoop.fs ChecksumException ChecksumException.

Prototype

public ChecksumException(String description, long pos) 

Source Link

Usage

From source file:org.apache.tez.engine.common.sort.impl.IFileInputStream.java

License:Apache License

private int doRead(byte[] b, int off, int len) throws IOException {

    // If we are trying to read past the end of data, just read
    // the left over data
    if (currentOffset + len > dataLength) {
        len = (int) dataLength - (int) currentOffset;
    }//  w  w w . j ava2s  .  c o m

    int bytesRead = in.read(b, off, len);

    if (bytesRead < 0) {
        throw new ChecksumException("Checksum Error", 0);
    }

    checksum(b, off, bytesRead);

    currentOffset += bytesRead;

    if (disableChecksumValidation) {
        return bytesRead;
    }

    if (currentOffset == dataLength) {
        // The last four bytes are checksum. Strip them and verify
        sum.update(buffer, 0, offset);
        csum = new byte[checksumSize];
        IOUtils.readFully(in, csum, 0, checksumSize);
        if (!sum.compare(csum, 0)) {
            throw new ChecksumException("Checksum Error", 0);
        }
    }
    return bytesRead;
}

---

From source file:org.apache.tez.engine.common.sort.impl.TezSpillRecord.java

License:Apache License

public TezSpillRecord(Path indexFileName, Configuration job, Checksum crc, String expectedIndexOwner)
        throws IOException {

    final FileSystem rfs = FileSystem.getLocal(job).getRaw();
    final FSDataInputStream in = rfs.open(indexFileName);
    try {//from  w  w w .  j  a v a 2  s.c  om
        final long length = rfs.getFileStatus(indexFileName).getLen();
        final int partitions = (int) length / Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;
        final int size = partitions * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH;

        buf = ByteBuffer.allocate(size);
        if (crc != null) {
            crc.reset();
            CheckedInputStream chk = new CheckedInputStream(in, crc);
            IOUtils.readFully(chk, buf.array(), 0, size);
            if (chk.getChecksum().getValue() != in.readLong()) {
                throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
            }
        } else {
            IOUtils.readFully(in, buf.array(), 0, size);
        }
        entries = buf.asLongBuffer();
    } finally {
        in.close();
    }
}

---

From source file:org.apache.tez.runtime.library.common.sort.impl.IFileInputStream.java

License:Apache License

private int doRead(byte[] b, int off, int len) throws IOException {

    // If we are trying to read past the end of data, just read
    // the left over data
    int origLen = len;
    if (currentOffset + len > dataLength) {
        len = (int) (dataLength - currentOffset);
    }/*from   ww  w  .j  a v a 2s .  c  o  m*/

    int bytesRead = in.read(b, off, len);

    if (bytesRead < 0) {
        String mesg = " CurrentOffset=" + currentOffset + ", offset=" + offset + ", off=" + off
                + ", dataLength=" + dataLength + ", origLen=" + origLen + ", len=" + len + ", length=" + length
                + ", checksumSize=" + checksumSize;
        LOG.info(mesg);
        throw new ChecksumException("Checksum Error: " + mesg, 0);
    }

    checksum(b, off, bytesRead);

    currentOffset += bytesRead;

    if (disableChecksumValidation) {
        return bytesRead;
    }

    if (currentOffset == dataLength) {
        // The last four bytes are checksum. Strip them and verify
        sum.update(buffer, 0, offset);
        csum = new byte[checksumSize];
        IOUtils.readFully(in, csum, 0, checksumSize);
        if (!sum.compare(csum, 0)) {
            String mesg = "CurrentOffset=" + currentOffset + ", off=" + offset + ", dataLength=" + dataLength
                    + ", origLen=" + origLen + ", len=" + len + ", length=" + length + ", checksumSize="
                    + checksumSize + ", csum=" + Arrays.toString(csum) + ", sum=" + sum;
            LOG.info(mesg);

            throw new ChecksumException("Checksum Error: " + mesg, 0);
        }
    }
    return bytesRead;
}

---