Example usage for org.apache.hadoop.util PureJavaCrc32 PureJavaCrc32

Introduction

In this page you can find the example usage for org.apache.hadoop.util PureJavaCrc32 PureJavaCrc32.

Prototype

public PureJavaCrc32() 

Source Link

Document

Create a new PureJavaCrc32 object.

Usage

From source file:com.cloudera.recordservice.examples.terasort.GenSort.java

License:Apache License

public static void outputRecords(OutputStream out, boolean useAscii, Unsigned16 firstRecordNumber,
        Unsigned16 recordsToGenerate, Unsigned16 checksum) throws IOException {
    byte[] row = new byte[100];
    Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
    Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
    Checksum crc = new PureJavaCrc32();
    Unsigned16 tmp = new Unsigned16();
    lastRecordNumber.add(recordsToGenerate);
    Unsigned16 ONE = new Unsigned16(1);
    Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
    while (!recordNumber.equals(lastRecordNumber)) {
        Random16.nextRand(rand);//from   w  w w.j  a  v a  2  s .c om
        if (useAscii) {
            generateAsciiRecord(row, rand, recordNumber);
        } else {
            generateRecord(row, rand, recordNumber);
        }
        if (checksum != null) {
            crc.reset();
            crc.update(row, 0, row.length);
            tmp.set(crc.getValue());
            checksum.add(tmp);
        }
        recordNumber.add(ONE);
        out.write(row);
    }
}

---

From source file:com.subspace.network.Util.java

License:Open Source License

public static int CRC32(byte[] data) {
    PureJavaCrc32 crc32Fast = new PureJavaCrc32();
    crc32Fast.update(data, 0, data.length);

    return (int) crc32Fast.getValue();
}

---

From source file:org.apache.tez.auxservices.TestShuffleHandler.java

License:Apache License

private static void createIndexFile(File indexFile, Configuration conf) throws IOException {
    if (indexFile.exists()) {
        System.out.println("Deleting existing file");
        indexFile.delete();//  w w  w. j av a2  s  . c o  m
    }
    indexFile.createNewFile();
    FSDataOutputStream output = FileSystem.getLocal(conf).getRaw()
            .append(new Path(indexFile.getAbsolutePath()));
    Checksum crc = new PureJavaCrc32();
    crc.reset();
    CheckedOutputStream chk = new CheckedOutputStream(output, crc);
    String msg = "Writing new index file. This file will be used only " + "for the testing.";
    chk.write(Arrays.copyOf(msg.getBytes(), MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
    output.writeLong(chk.getChecksum().getValue());
    output.close();
}

---

From source file:org.apache.tez.engine.common.sort.impl.TezSpillRecord.java

License:Apache License

public TezSpillRecord(Path indexFileName, Configuration job, String expectedIndexOwner) throws IOException {
    this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}

---

From source file:org.apache.tez.engine.common.sort.impl.TezSpillRecord.java

License:Apache License

/**
 * Write this spill record to the location provided.
 *///  ww  w.  j  a  v  a  2 s  .co  m
public void writeToFile(Path loc, Configuration job) throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
}

---