Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.lib.DynamicInputFormat.java

License:Apache License

private List<DynamicInputChunk> splitCopyListingIntoChunksWithShuffle(JobContext context) throws IOException {

    final Configuration configuration = HadoopCompat.getConfiguration(context);
    int numRecords = getNumberOfRecords(configuration);
    int numMaps = getNumMapTasks(configuration);
    // Number of chunks each map will process, on average.
    int splitRatio = getListingSplitRatio(configuration, numMaps, numRecords);
    validateNumChunksUsing(splitRatio, numMaps);

    int numEntriesPerChunk = (int) Math.ceil((float) numRecords / (splitRatio * numMaps));
    DistCpUtils.publish(HadoopCompat.getConfiguration(context), CONF_LABEL_NUM_ENTRIES_PER_CHUNK,
            numEntriesPerChunk);//ww w .  j  a  v  a2 s. co m

    final int nChunksTotal = (int) Math.ceil((float) numRecords / numEntriesPerChunk);
    int nChunksOpenAtOnce = Math.min(N_CHUNKS_OPEN_AT_ONCE_DEFAULT, nChunksTotal);

    Path listingPath = getListingFilePath(configuration);
    SequenceFile.Reader reader = new SequenceFile.Reader(listingPath.getFileSystem(configuration), listingPath,
            configuration);

    List<DynamicInputChunk> openChunks = new ArrayList<DynamicInputChunk>();

    List<DynamicInputChunk> chunksFinal = new ArrayList<DynamicInputChunk>();

    FileStatus fileStatus = new FileStatus();
    Text relPath = new Text();
    int recordCounter = 0;
    int chunkCount = 0;
    DynamicInputChunkSet chunkSet = new DynamicInputChunkSet(configuration);

    try {

        while (reader.next(relPath, fileStatus)) {
            if (recordCounter % (nChunksOpenAtOnce * numEntriesPerChunk) == 0) {
                // All chunks full. Create new chunk-set.
                closeAll(openChunks);
                chunksFinal.addAll(openChunks);

                openChunks = createChunks(chunkSet, chunkCount, nChunksTotal, nChunksOpenAtOnce);

                chunkCount += openChunks.size();

                nChunksOpenAtOnce = openChunks.size();
                recordCounter = 0;
            }

            // Shuffle into open chunks.
            openChunks.get(recordCounter % nChunksOpenAtOnce).write(relPath, fileStatus);
            ++recordCounter;
        }

    } finally {
        closeAll(openChunks);
        chunksFinal.addAll(openChunks);
        IOUtils.closeStream(reader);
    }

    LOG.info("Number of dynamic-chunk-files created: " + chunksFinal.size());
    return chunksFinal;
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestUniformSizeInputFormat.java

License:Apache License

private void checkSplits(Path listFile, List<InputSplit> splits) throws IOException {
    long lastEnd = 0;

    //Verify if each split's start is matching with the previous end and
    //we are not missing anything
    for (InputSplit split : splits) {
        FileSplit fileSplit = (FileSplit) split;
        long start = fileSplit.getStart();
        Assert.assertEquals(lastEnd, start);
        lastEnd = start + fileSplit.getLength();
    }//from  w  w w  .  j  ava 2s . c om

    //Verify there is nothing more to read from the input file
    SequenceFile.Reader reader = new SequenceFile.Reader(cluster.getFileSystem(), listFile,
            cluster.getFileSystem().getConf());
    try {
        reader.seek(lastEnd);
        FileStatus srcFileStatus = new FileStatus();
        Text srcRelPath = new Text();
        Assert.assertFalse(reader.next(srcRelPath, srcFileStatus));
    } finally {
        IOUtils.closeStream(reader);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.UniformSizeInputFormat.java

License:Apache License

private List<InputSplit> getSplits(Configuration configuration, int numSplits, long totalSizeBytes)
        throws IOException {
    List<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
    long nBytesPerSplit = (long) Math.ceil(totalSizeBytes * 1.0 / numSplits);

    FileStatus srcFileStatus = new FileStatus();
    Text srcRelPath = new Text();
    long currentSplitSize = 0;
    long lastSplitStart = 0;
    long lastPosition = 0;

    final Path listingFilePath = getListingFilePath(configuration);

    if (LOG.isDebugEnabled()) {
        LOG.debug("Average bytes per map: " + nBytesPerSplit + ", Number of maps: " + numSplits
                + ", total size: " + totalSizeBytes);
    }/*w  w w .ja va2 s. c om*/
    SequenceFile.Reader reader = null;
    try {
        reader = getListingFileReader(configuration);
        while (reader.next(srcRelPath, srcFileStatus)) {
            // If adding the current file would cause the bytes per map to exceed
            // limit. Add the current file to new split
            if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
                FileSplit split = new FileSplit(listingFilePath, lastSplitStart, lastPosition - lastSplitStart,
                        null);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Creating split : " + split + ", bytes in split: " + currentSplitSize);
                }
                splits.add(split);
                lastSplitStart = lastPosition;
                currentSplitSize = 0;
            }
            currentSplitSize += srcFileStatus.getLen();
            lastPosition = reader.getPosition();
        }
        if (lastPosition > lastSplitStart) {
            FileSplit split = new FileSplit(listingFilePath, lastSplitStart, lastPosition - lastSplitStart,
                    null);
            if (LOG.isDebugEnabled()) {
                LOG.info("Creating split : " + split + ", bytes in split: " + currentSplitSize);
            }
            splits.add(split);
        }

    } finally {
        IOUtils.closeStream(reader);
    }

    return splits;
}

---

From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java

License:Apache License

@Test
public void testBuildListingForSingleFile() {
    FileSystem fs = null;//from  www  . j  av a2s. c  o  m
    String testRootString = "/singleFileListing";
    Path testRoot = new Path(testRootString);
    SequenceFile.Reader reader = null;
    try {
        fs = FileSystem.get(getConf());
        if (fs.exists(testRoot))
            TestDistCpUtils.delete(fs, testRootString);

        Path sourceFile = new Path(testRoot, "/source/foo/bar/source.txt");
        Path decoyFile = new Path(testRoot, "/target/moo/source.txt");
        Path targetFile = new Path(testRoot, "/target/moo/target.txt");

        TestDistCpUtils.createFile(fs, sourceFile.toString());
        TestDistCpUtils.createFile(fs, decoyFile.toString());
        TestDistCpUtils.createFile(fs, targetFile.toString());

        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(sourceFile);

        DistCpOptions options = new DistCpOptions(srcPaths, targetFile);
        CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);

        final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
        listing.buildListing(listFile, options);

        reader = new SequenceFile.Reader(fs, listFile, getConf());
        FileStatus fileStatus = new FileStatus();
        Text relativePath = new Text();
        Assert.assertTrue(reader.next(relativePath, fileStatus));
        Assert.assertTrue(relativePath.toString().equals(""));
    } catch (Exception e) {
        Assert.fail("Unexpected exception encountered.");
        LOG.error("Unexpected exception: ", e);
    } finally {
        TestDistCpUtils.delete(fs, testRootString);
        IOUtils.closeStream(reader);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestFileBasedCopyListing.java

License:Apache License

private void checkResult(Path listFile, int count) throws IOException {
    if (count == 0) {
        return;//from w  w  w .j av  a 2 s  .c om
    }

    int recCount = 0;
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, listFile, config);
    try {
        Text relPath = new Text();
        FileStatus fileStatus = new FileStatus();
        while (reader.next(relPath, fileStatus)) {
            Assert.assertEquals(fileStatus.getPath().toUri().getPath(), map.get(relPath.toString()));
            recCount++;
        }
    } finally {
        IOUtils.closeStream(reader);
    }
    Assert.assertEquals(recCount, count);
}

---

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

public static void createFile(FileSystem fs, String filePath) throws IOException {
    OutputStream out = fs.create(new Path(filePath));
    IOUtils.closeStream(out);
}

---

From source file:com.inmobi.conduit.distcp.tools.util.TestThrottledInputStream.java

License:Apache License

private long copyAndAssert(File tmpFile, File outFile, long maxBandwidth, float factor, int sleepTime, CB flag)
        throws IOException {
    long bandwidth;
    ThrottledInputStream in;//w w w  .  jav a 2  s. c  om
    long maxBPS = (long) (maxBandwidth / factor);

    if (maxBandwidth == 0) {
        in = new ThrottledInputStream(new FileInputStream(tmpFile));
    } else {
        in = new ThrottledInputStream(new FileInputStream(tmpFile), maxBPS);
    }
    OutputStream out = new FileOutputStream(outFile);
    try {
        if (flag == CB.BUFFER) {
            copyBytes(in, out, BUFF_SIZE);
        } else if (flag == CB.BUFF_OFFSET) {
            copyBytesWithOffset(in, out, BUFF_SIZE);
        } else {
            copyByteByByte(in, out);
        }

        LOG.info(in);
        bandwidth = in.getBytesPerSec();
        Assert.assertEquals(in.getTotalBytesRead(), tmpFile.length());
        Assert.assertTrue(in.getBytesPerSec() > maxBandwidth / (factor * 1.2));
        Assert.assertTrue(in.getTotalSleepTime() > sleepTime || in.getBytesPerSec() <= maxBPS);
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
    return bandwidth;
}

---

From source file:com.inmobi.conduit.distcp.tools.util.TestThrottledInputStream.java

License:Apache License

private void writeToFile(File tmpFile, long sizeInKB) throws IOException {
    OutputStream out = new FileOutputStream(tmpFile);
    try {/*  www  . j  a v  a2  s  . c  om*/
        byte[] buffer = new byte[1024];
        for (long index = 0; index < sizeInKB; index++) {
            out.write(buffer);
        }
    } finally {
        IOUtils.closeStream(out);
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java

License:Apache License

public Writable readWritable(Writable writable) throws IOException {
    DataInputStream dis = null;/*from w w  w  .  j  a va  2 s  .co  m*/
    try {
        ByteArrayInputStream bais = new ByteArrayInputStream(in.readBytes());
        dis = new DataInputStream(bais);
        String className = WritableUtils.readString(dis);
        if (writable == null) {
            try {
                Class<? extends Writable> cls = conf.getClassByName(className).asSubclass(Writable.class);
                writable = (Writable) ReflectionUtils.newInstance(cls, conf);
            } catch (ClassNotFoundException e) {
                throw new IOException(e);
            }
        } else if (!writable.getClass().getName().equals(className)) {
            throw new IOException("wrong Writable class given");
        }
        writable.readFields(dis);
        dis.close();
        dis = null;
        return writable;
    } finally {
        IOUtils.closeStream(dis);
    }
}

---

From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java

License:Apache License

public void writeWritable(Writable w) throws IOException {
    DataOutputStream dos = null;/*  w ww  . ja v a  2  s . co  m*/
    try {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        dos = new DataOutputStream(baos);
        WritableUtils.writeString(dos, w.getClass().getName());
        w.write(dos);
        out.writeBytes(baos.toByteArray(), RType.WRITABLE.code);
        dos.close();
        dos = null;
    } finally {
        IOUtils.closeStream(dos);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »