Example usage for org.apache.hadoop.hdfs DistributedFileSystem getClient

List of usage examples for org.apache.hadoop.hdfs DistributedFileSystem getClient

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.hdfs.*
  5. DistributedFileSystem
  6. getClient

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DistributedFileSystem getClient.

Prototype

@InterfaceAudience.Private
    @VisibleForTesting
    public DFSClient getClient()

Source Link

Usage

From source file:com.splunk.shuttl.prototype.symlink.BucketBlockSymlinkPrototypeTest.java

License:Apache License

private void createSymlinkToPathInDir(Path path, File dir) throws IOException {
    File fileInDir = new File(dir, path.getName());

    DistributedFileSystem dfs = (DistributedFileSystem) hadoopFileSystem;
    ClientProtocol namenode = dfs.getClient().namenode;
    String pathOnHadoop = path.toUri().getPath();
    LocatedBlocks blockLocations = namenode.getBlockLocations(pathOnHadoop, 0, Long.MAX_VALUE);
    List<LocatedBlock> locatedBlocks = blockLocations.getLocatedBlocks();
    if (!locatedBlocks.isEmpty()) {
        doSymlinkPathInDir(fileInDir, blockLocations, locatedBlocks);
    } else {/*  w  w w  . j av  a  2s . com*/
        // Means that they don't have a block and that they are empty files. Just
        // create them.
        assertTrue(fileInDir.createNewFile());
    }
}

From source file:io.hops.erasure_coding.Decoder.java

License:Apache License

public void recoverParityBlockToFile(FileSystem srcFs, Path srcPath, FileSystem parityFs, Path parityPath,
        long blockSize, long blockOffset, File localBlockFile, Context context)
        throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) srcFs;
    long crc32 = dfs.getClient().getBlockChecksum(parityPath.toUri().toString(),
            (int) (blockOffset / blockSize));
    OutputStream out = null;/*  w w w .j a  v  a  2 s .c  o m*/
    try {
        out = new FileOutputStream(localBlockFile);
        fixErasedBlock(srcFs, srcPath, parityFs, parityPath, false, blockSize, blockOffset, blockSize, false,
                out, context, false, crc32);
    } finally {
        if (out != null) {
            out.close();
        }
    }
}

From source file:io.hops.erasure_coding.Decoder.java

License:Apache License

/**
 * Recovers a corrupt block to local file.
 *
 * @param srcFs/*from w  w  w  .j  av  a  2 s  .com*/
 *     The filesystem containing the source file.
 * @param srcPath
 *     The damaged source file.
 * @param parityPath
 *     The filesystem containing the parity file. This could be
 *     different from fs in case the parity file is part of a HAR archive.
 * @param blockSize
 *     The block size of the file.
 * @param blockOffset
 *     Known location of error in the source file. There could
 *     be additional errors in the source file that are discovered during
 *     the decode process.
 * @param localBlockFile
 *     The file to write the block to.
 * @param limit
 *     The maximum number of bytes to be written out.
 *     This is to prevent writing beyond the end of the file.
 */
public void recoverBlockToFile(FileSystem srcFs, Path srcPath, FileSystem parityFs, Path parityPath,
        long blockSize, long blockOffset, File localBlockFile, long limit, Context context)
        throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) srcFs;
    long crc32 = dfs.getClient().getBlockChecksum(srcPath.toUri().getPath(), (int) (blockOffset / blockSize));
    OutputStream out = null;
    try {
        out = new FileOutputStream(localBlockFile);
        fixErasedBlock(srcFs, srcPath, parityFs, parityPath, true, blockSize, blockOffset, limit, false, out,
                context, false, crc32);
    } finally {
        if (out != null) {
            out.close();
        }
    }
}

From source file:io.hops.erasure_coding.Encoder.java

License:Apache License

private void sendChecksums(DistributedFileSystem dfs, Path file, Checksum[] checksums, int stripe, int length)
        throws IOException {
    if (checksums == null) {
        return;// w ww.ja  va2 s.  co  m
    }
    DFSClient dfsClient = dfs.getClient();
    int firstBlockIndex = stripe * length;
    for (int i = 0; i < length; i++) {
        int blockIndex = firstBlockIndex + i;
        dfsClient.addBlockChecksum(file.toUri().getPath(), blockIndex, checksums[i].getValue());
    }
}

From source file:io.hops.erasure_coding.TestBlockReconstructor.java

License:Apache License

@Test
public void testSourceBlockRepair() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();
    TestDfsClient testDfsClient = new TestDfsClient(getConfig());
    testDfsClient.injectIntoDfs(dfs);/*from  w ww. j a v a 2  s  .c  o m*/
    FileStatus testFileStatus = dfs.getFileStatus(testFile);

    String path = testFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    List<LocatedBlock> lostBlocks = new ArrayList<LocatedBlock>();
    lostBlocks.add(lb);
    LocatedBlocks locatedBlocks = new LocatedBlocks(0, false, lostBlocks, null, true);
    testDfsClient.setMissingLocatedBlocks(locatedBlocks);

    LocatedBlocks missingBlocks = new LocatedBlocks(testFileStatus.getLen(), false,
            new ArrayList<LocatedBlock>(), null, true);
    missingBlocks.getLocatedBlocks().add(lb);
    BlockReconstructor blockReconstructor = new BlockReconstructor(conf);
    Decoder decoder = new Decoder(conf, Util.getCodec(Util.Codecs.SRC));
    blockReconstructor.processFile(testFile, testParityFile, missingBlocks, decoder, null);

    // Block is recovered to the same data node so no need to wait for the block report
    try {
        FSDataInputStream in = dfs.open(testFile);
        byte[] buff = new byte[TEST_BLOCK_COUNT * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
    } catch (BlockMissingException e) {
        LOG.error("Reading failed", e);
        fail("Repair failed. Missing a block.");
    }
}

From source file:io.hops.erasure_coding.TestBlockReconstructor.java

License:Apache License

@Test
public void testParityBlockRepair() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();
    TestDfsClient testDfsClient = new TestDfsClient(getConfig());
    testDfsClient.injectIntoDfs(dfs);//from ww  w. j  a va  2 s  .c om
    FileStatus parityFileStatus = dfs.getFileStatus(testParityFile);

    String path = parityFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (parityFileStatus.getLen() / parityFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    List<LocatedBlock> lostBlocks = new ArrayList<LocatedBlock>();
    lostBlocks.add(lb);
    LocatedBlocks locatedBlocks = new LocatedBlocks(0, false, lostBlocks, null, true);
    testDfsClient.setMissingLocatedBlocks(locatedBlocks);

    LocatedBlocks missingBlocks = new LocatedBlocks(parityFileStatus.getLen(), false,
            new ArrayList<LocatedBlock>(), null, true);
    missingBlocks.getLocatedBlocks().add(lb);
    BlockReconstructor blockReconstructor = new BlockReconstructor(conf);
    Decoder decoder = new Decoder(conf, Util.getCodec(Util.Codecs.SRC));
    blockReconstructor.processParityFile(testFile, testParityFile, missingBlocks, decoder, null);

    // Block is recovered to the same data node so no need to wait for the block report
    try {
        FSDataInputStream in = dfs.open(testParityFile);
        byte[] buff = new byte[DFS_TEST_BLOCK_SIZE * codec.parityLength];
        in.readFully(0, buff);
    } catch (BlockMissingException e) {
        LOG.error("Reading failed", e);
        fail("Repair failed. Missing a block.");
    }
}

From source file:io.hops.erasure_coding.TestErasureCodingManagerEndless.java

License:Apache License

@Ignore
public void endlessSourceTest() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();

    Codec.initializeCodecs(getConfig());
    EncodingPolicy policy = new EncodingPolicy("src", (short) 1);
    Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy);
    FileStatus testFileStatus = dfs.getFileStatus(testFile);

    while (!dfs.getEncodingStatus(testFile.toUri().getPath()).isEncoded()) {
        try {//from w  ww.j  av a2s . c om
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.error("Wait for encoding thread was interrupted.");
        }
    }

    EncodingStatus status = dfs.getEncodingStatus(testFile.toUri().getPath());
    Path parityPath = new Path("/parity/" + status.getParityFileName());
    FileStatus parityStatus = dfs.getFileStatus(parityPath);
    assertEquals(parityStatus.getLen(), TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE);
    try {
        FSDataInputStream in = dfs.open(parityPath);
        byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
    } catch (BlockMissingException e) {
        LOG.error("Reading parity failed", e);
        fail("Parity could not be read.");
    }

    String path = testFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    LOG.info("Loosing block " + lb.toString());

    EncodingStatus lastStatus = null;
    while (true) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Was interrupted", e);
        }
        EncodingStatus status2 = dfs.getEncodingStatus(testFile.toUri().getPath());
        if (status2.equals(lastStatus) == false) {
            LOG.info("New status is " + status2.getStatus());
            lastStatus = status2;
        }
    }
}

From source file:io.hops.erasure_coding.TestErasureCodingManagerEndless.java

License:Apache License

@Ignore
public void endlessParityTest() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();

    Codec.initializeCodecs(getConfig());
    EncodingPolicy policy = new EncodingPolicy("src", (short) 1);
    Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy);
    FileStatus testFileStatus = dfs.getFileStatus(testFile);

    while (!dfs.getEncodingStatus(testFile.toUri().getPath()).isEncoded()) {
        try {//from www  . j  a  v  a  2  s.  c o m
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.error("Wait for encoding thread was interrupted.");
        }
    }

    EncodingStatus status = dfs.getEncodingStatus(testFile.toUri().getPath());
    Path parityPath = new Path("/parity/" + status.getParityFileName());
    FileStatus parityStatus = dfs.getFileStatus(parityPath);
    assertEquals(parityStatus.getLen(), TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE);
    try {
        FSDataInputStream in = dfs.open(parityPath);
        byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
    } catch (BlockMissingException e) {
        LOG.error("Reading parity failed", e);
        fail("Parity could not be read.");
    }

    int blockToLoose = new Random(seed).nextInt((int) (parityStatus.getLen() / parityStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(parityPath.toUri().getPath(), 0, Long.MAX_VALUE)
            .get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    LOG.info("Loosing block " + lb.toString());

    try {
        FSDataInputStream in = dfs.open(parityPath);
        byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
        fail("Successfully read parity file which should have been broken.");
    } catch (BlockMissingException e) {
    }

    EncodingStatus lastStatus = null;
    while (true) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Was interrupted", e);
        }
        EncodingStatus status2 = dfs.getEncodingStatus(testFile.toUri().getPath());
        if (status2.equals(lastStatus) == false) {
            LOG.info("New status is " + status2);
            lastStatus = status2;
        }
    }
}

From source file:io.hops.erasure_coding.TestMapReduceBlockRepairManager.java

License:Apache License

@Test
public void testBlockRepair() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();
    TestDfsClient testDfsClient = new TestDfsClient(getConfig());
    testDfsClient.injectIntoDfs(dfs);//w ww  .  ja va 2s  .  c o m

    MapReduceEncodingManager encodingManager = new MapReduceEncodingManager(conf);

    Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE);
    Codec.initializeCodecs(conf);
    FileStatus testFileStatus = dfs.getFileStatus(testFile);
    EncodingPolicy policy = new EncodingPolicy("src", (short) 1);
    encodingManager.encodeFile(policy, testFile, parityFile);

    // Busy waiting until the encoding is done
    while (encodingManager.computeReports().size() > 0) {
        ;
    }

    String path = testFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    List<LocatedBlock> lostBlocks = new ArrayList<LocatedBlock>();
    lostBlocks.add(lb);
    LocatedBlocks locatedBlocks = new LocatedBlocks(0, false, lostBlocks, null, true);
    testDfsClient.setMissingLocatedBlocks(locatedBlocks);
    LOG.info("Loosing block " + lb.toString());
    getCluster().triggerBlockReports();

    MapReduceBlockRepairManager repairManager = new MapReduceBlockRepairManager(conf);
    repairManager.repairSourceBlocks("src", testFile, parityFile);

    while (true) {
        List<Report> reports = repairManager.computeReports();
        if (reports.size() == 0) {
            break;
        }
        LOG.info(reports.get(0).getStatus());
        System.out.println("WAIT");
        Thread.sleep(1000);
    }

    try {
        FSDataInputStream in = dfs.open(testFile);
        byte[] buff = new byte[TEST_BLOCK_COUNT * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
    } catch (BlockMissingException e) {
        fail("Repair failed. Missing a block.");
    }
}

From source file:io.hops.erasure_coding.TestMapReduceBlockRepairManager.java

License:Apache License

@Test
public void testCorruptedRepair() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();
    TestDfsClient testDfsClient = new TestDfsClient(getConfig());
    testDfsClient.injectIntoDfs(dfs);//from   w ww . j  av  a  2 s .  c  o m

    MapReduceEncodingManager encodingManager = new MapReduceEncodingManager(conf);

    Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE);
    Codec.initializeCodecs(conf);
    FileStatus testFileStatus = dfs.getFileStatus(testFile);
    EncodingPolicy policy = new EncodingPolicy("src", (short) 1);
    encodingManager.encodeFile(policy, testFile, parityFile);

    // Busy waiting until the encoding is done
    while (encodingManager.computeReports().size() > 0) {
        ;
    }

    String path = testFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    List<LocatedBlock> lostBlocks = new ArrayList<LocatedBlock>();
    lostBlocks.add(lb);
    LocatedBlocks locatedBlocks = new LocatedBlocks(0, false, lostBlocks, null, true);
    testDfsClient.setMissingLocatedBlocks(locatedBlocks);
    LOG.info("Loosing block " + lb.toString());
    getCluster().triggerBlockReports();

    dfs.getClient().addBlockChecksum(testFile.toUri().getPath(),
            (int) (lb.getStartOffset() / lb.getBlockSize()), 0);

    MapReduceBlockRepairManager repairManager = new MapReduceBlockRepairManager(conf);
    repairManager.repairSourceBlocks("src", testFile, parityFile);

    while (true) {
        List<Report> reports = repairManager.computeReports();
        if (reports.size() == 0) {
            break;
        }
        LOG.info(reports.get(0).getStatus());
        System.out.println("WAIT");
        Thread.sleep(1000);
    }

    try {
        FSDataInputStream in = dfs.open(testFile);
        byte[] buff = new byte[TEST_BLOCK_COUNT * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
        fail("Repair succeeded with bogus checksum.");
    } catch (BlockMissingException e) {
    }
}