Example usage for org.apache.hadoop.fs.contract ContractTestUtils generateTestFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs.contract ContractTestUtils generateTestFile.

Prototype

public static long generateTestFile(FileSystem fs, Path path, final long size, final int bufferLen,
        final int modulus) throws IOException 

Source Link

Document

Generates test data of the given size according to some specific pattern and writes it to the provided output file.

Usage

From source file:com.aliyun.fs.oss.TestAliyunOSSInputStream.java

License:Apache License

@Test
public void testSeekFile() throws Exception {
    Path smallSeekFile = setPath("/test/smallSeekFile.txt");
    long size = 5 * 1024 * 1024;

    ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255);
    LOG.info("5MB file created: smallSeekFile.txt");

    FSDataInputStream instream = this.fs.open(smallSeekFile);
    int seekTimes = 5;
    LOG.info("multiple fold position seeking test...:");
    for (int i = 0; i < seekTimes; i++) {
        long pos = size / (seekTimes - i) - 1;
        LOG.info("begin seeking for pos: " + pos);
        instream.seek(pos);// w  w w  . j  a va 2s  .c o m
        assertTrue("expected position at:" + pos + ", but got:" + instream.getPos(), instream.getPos() == pos);
        LOG.info("completed seeking at pos: " + instream.getPos());
    }
    LOG.info("random position seeking test...:");
    Random rand = new Random();
    for (int i = 0; i < seekTimes; i++) {
        long pos = Math.abs(rand.nextLong()) % size;
        LOG.info("begin seeking for pos: " + pos);
        instream.seek(pos);
        assertTrue("expected position at:" + pos + ", but got:" + instream.getPos(), instream.getPos() == pos);
        LOG.info("completed seeking at pos: " + instream.getPos());
    }
    IOUtils.closeStream(instream);
}

---

From source file:com.aliyun.fs.oss.TestAliyunOSSInputStream.java

License:Apache License

@Test
public void testReadFile() throws Exception {
    final int bufLen = 256;
    final int sizeFlag = 5;
    String filename = "readTestFile_" + sizeFlag + ".txt";
    Path readTestFile = setPath("/test/" + filename);
    long size = sizeFlag * 1024 * 1024;

    ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255);
    LOG.info(sizeFlag + "MB file created: /test/" + filename);

    FSDataInputStream instream = this.fs.open(readTestFile);
    byte[] buf = new byte[bufLen];
    long bytesRead = 0;
    while (bytesRead < size) {
        int bytes;
        if (size - bytesRead < bufLen) {
            int remaining = (int) (size - bytesRead);
            bytes = instream.read(buf, 0, remaining);
        } else {//from   www  . j  a v a  2s .c  o  m
            bytes = instream.read(buf, 0, bufLen);
        }
        bytesRead += bytes;

        if (bytesRead % (1024 * 1024) == 0) {
            int available = instream.available();
            int remaining = (int) (size - bytesRead);
            assertTrue("expected remaining:" + remaining + ", but got:" + available, remaining == available);
            LOG.info("Bytes read: " + Math.round((double) bytesRead / (1024 * 1024)) + " MB");
        }
    }
    assertTrue(instream.available() == 0);
    IOUtils.closeStream(instream);
}

---