Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_CHECKSUM_TYPE

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_CHECKSUM_TYPE_KEY.

Prototype

String DFS_CHECKSUM_TYPE_KEY

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_CHECKSUM_TYPE_KEY.

Click Source Link

Usage

From source file:backup.datanode.DataNodeRestoreProcessor.java

License:Apache License

public DataNodeRestoreProcessor(Configuration conf, DataNode datanode) throws Exception {
    _closer = Closer.create();/*from   w w w.ja v  a 2s  .  c  om*/
    _datanode = datanode;
    _restoreThroughput = Metrics.METRICS.meter(RESTORE_THROUGHPUT);
    _bytesPerChecksum = conf.getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
            DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
    _checksumType = Type
            .valueOf(conf.get(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT));
    int threads = conf.getInt(DFS_BACKUP_DATANODE_RESTORE_BLOCK_HANDLER_COUNT_KEY,
            DFS_BACKUP_DATANODE_RESTORE_BLOCK_HANDLER_COUNT_DEFAULT);
    long pauseOnError = conf.getLong(DFS_BACKUP_DATANODE_RESTORE_ERROR_PAUSE_KEY,
            DFS_BACKUP_DATANODE_RESTORE_ERROR_PAUSE_DEFAULT);
    _backupStore = _closer.register(BackupStore.create(BackupUtil.convert(conf)));
    _restoreBlocks = new ArrayBlockingQueue<>(threads);
    _executorService = Executors.newCachedThreadPool();
    _closer.register((Closeable) () -> _executorService.shutdownNow());
    for (int t = 0; t < threads; t++) {
        _executorService.submit(Executable.createDaemon(LOG, pauseOnError, _running, () -> restoreBlocks()));
    }
}

From source file:com.mellanox.r4h.TestFSOutputSummer.java

License:Apache License

private void doTestFSOutputSummer(String checksumType) throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM);
    conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, checksumType);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build();
    fileSys = cluster.getFileSystem();//from w ww  .ja va 2  s  .c  o m
    try {
        Path file = new Path("try.dat");
        Random rand = new Random(seed);
        rand.nextBytes(expected);
        writeFile1(file);
        writeFile2(file);
        writeFile3(file);
    } finally {
        fileSys.close();
        cluster.shutdown();
    }
}

From source file:com.mellanox.r4h.TestFSOutputSummer.java

License:Apache License

@Test
public void TestDFSCheckSumType() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM);
    conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "NULL");
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build();
    fileSys = cluster.getFileSystem();/*from   w w w. j  a  va2 s. c  o  m*/
    try {
        Path file = new Path("try.dat");
        Random rand = new Random(seed);
        rand.nextBytes(expected);
        writeFile1(file);
    } finally {
        fileSys.close();
        cluster.shutdown();
    }
}

Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_CHECKSUM_TYPE_KEY

Introduction

Prototype

Usage