List of usage examples for org.apache.hadoop.io IOUtils readFully
public static void readFully(InputStream in, byte[] buf, int off, int len) throws IOException
From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryArrayZigZarByteReader.java
License:Apache License
public void ensureDecompress() throws IOException { // if (compressAlgo != null) { org.apache.hadoop.io.compress.Decompressor decompressor = this.compressAlgo.getDecompressor(); InputStream is = this.compressAlgo.createDecompressionStream(inBuf, decompressor, 0); ByteBuffer buf = ByteBuffer.allocate(decompressedSize); IOUtils.readFully(is, buf.array(), 0, buf.capacity()); is.close();// ww w . j av a 2s .com this.compressAlgo.returnDecompressor(decompressor); inBuf.reset(buf.array(), offset, buf.capacity()); }
From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryBitPackingZigZarIntReader.java
License:Apache License
public void ensureDecompress() throws IOException { org.apache.hadoop.io.compress.Decompressor decompressor = this.compressAlgo.getDecompressor(); InputStream is = this.compressAlgo.createDecompressionStream(inBuf, decompressor, 0); ByteBuffer buf = ByteBuffer.allocate(decompressedSize); IOUtils.readFully(is, buf.array(), 0, buf.capacity()); is.close();/*from w w w . j a v a 2 s .c o m*/ this.compressAlgo.returnDecompressor(decompressor); inBuf.reset(buf.array(), offset, buf.capacity()); }
From source file:cn.ac.ncic.mastiff.io.coding.DeltaBinaryPackingStringReader.java
License:Apache License
public void ensureDecompress() throws IOException { org.apache.hadoop.io.compress.Decompressor decompressor = this.compressAlgo.getDecompressor(); InputStream is = this.compressAlgo.createDecompressionStream(inBuf, decompressor, 0); ByteBuffer buf = ByteBuffer.allocate(decompressedSize); // ByteBuffer buf = ByteBuffer.allocate(is.available()); IOUtils.readFully(is, buf.array(), 0, buf.capacity()); is.close();//from ww w. ja v a 2 s.co m this.compressAlgo.returnDecompressor(decompressor); inBuf.reset(buf.array(), offset, buf.capacity()); }
From source file:cn.ac.ncic.mastiff.io.coding.MVDecoder.java
License:Apache License
public void ensureDecompress() throws IOException { if (compressAlgo != null && page == null) { org.apache.hadoop.io.compress.Decompressor decompressor = this.compressAlgo.getDecompressor(); InputStream is = this.compressAlgo.createDecompressionStream(inBuf, decompressor, 0); ByteBuffer buf = ByteBuffer.allocate(decompressedSize); IOUtils.readFully(is, buf.array(), 3 * Bytes.SIZEOF_INT, buf.capacity() - 3 * Bytes.SIZEOF_INT); is.close();/*from w ww.j a va 2 s. com*/ this.compressAlgo.returnDecompressor(decompressor); page = buf.array(); } }
From source file:cn.ac.ncic.mastiff.io.coding.RunLengthEncodingByteReader.java
License:Apache License
public void ensureDecompress() throws IOException { org.apache.hadoop.io.compress.Decompressor decompressor = this.compressAlgo.getDecompressor(); InputStream is = this.compressAlgo.createDecompressionStream(inBuf, decompressor, 0); ByteBuffer buf = ByteBuffer.allocate(decompressedSize); IOUtils.readFully(is, buf.array(), 0, buf.capacity()); is.close();//from ww w. j a v a 2 s . com this.compressAlgo.returnDecompressor(decompressor); inBuf.reset(buf.array(), offset, buf.capacity()); }
From source file:com.asakusafw.runtime.stage.temporary.TemporaryFile.java
License:Apache License
/** * Reads a string into appendable./*from w w w. j a v a 2 s .c o m*/ * @param input the target input * @param appendable the target appendable * @return the bytes read * @throws IOException if failed to read */ public static int readString(InputStream input, Appendable appendable) throws IOException { int length = 0; for (int i = 0; i < 4; i++) { int c = input.read(); if (c < 0) { return -1; } length = length << 8 | c; } byte[] bytes = new byte[length]; IOUtils.readFully(input, bytes, 0, length); appendable.append(new String(bytes, ENCODING)); return 4 + bytes.length; }
From source file:com.cloudera.hadoop.hdfs.nfs.rpc.RPCBuffer.java
License:Apache License
/** * This method will read an RPC request/response from InputStream and store * the resulting data in the buffer/* ww w. ja v a 2s. c o m*/ * * @param in * @throws IOException */ public static RPCBuffer from(InputStream in) throws IOException { List<byte[]> buffers = new ArrayList<byte[]>(1); byte[] header = new byte[4]; int size = 0; boolean last = false; while (!last) { readFully(in, header, 0, header.length); int raw = Bytes.toInt(header); size = raw; last = (size & RPC_LAST_FRAGEMANT) != 0; size &= RPC_SIZE_MASK; if (LOGGER.isDebugEnabled()) { LOGGER.debug("Read Header for Packet of Size " + size + ", last = " + last); } byte[] buffer = new byte[size]; IOUtils.readFully(in, buffer, 0, size); buffers.add(buffer); } byte[] buffer = Bytes.merge(buffers); // XXX this absolutely bad if (LOGGER.isDebugEnabled()) { LOGGER.debug("From InputStream " + Bytes.asHex(buffer)); } return new RPCBuffer(buffer, buffer.length); }
From source file:com.facebook.hive.orc.ReaderImpl.java
License:Open Source License
/** * Reads the file header (first 40 bytes) and checks if the first three characters are 'ORC'. *///from ww w .j ava 2s. c o m public static void checkIfORC(FileSystem fs, Path path) throws IOException { // hardcoded to 40 because "SEQ-org.apache.hadoop.hive.ql.io.RCFile", the header, is of 40 chars final int buffLen = 40; final byte header[] = new byte[buffLen]; final FSDataInputStream file = fs.open(path); final long fileLength = fs.getFileStatus(path).getLen(); int sizeToBeRead = buffLen; if (buffLen > fileLength) { sizeToBeRead = (int) fileLength; } IOUtils.readFully(file, header, 0, sizeToBeRead); file.close(); final String headerString = new String(header); if (headerString.startsWith("ORC")) { LOG.error("Error while parsing the footer of the file : " + path); } else { throw new NotAnORCFileException("Input file = " + path + " , header = " + headerString); } }
From source file:com.marcolotz.lung.io.inputFormat.MultipleFilesRecordReader.java
License:Creative Commons License
/** * <p>//from www .j a va 2s . c o m * If the file has not already been read, this reads it into memory, so that * a call to getCurrentValue() will return the entire contents of this file * as Text, and getCurrentKey() will return the qualified path to this file * as Text. Then, returns true. If it has already been read, then returns * false without updating any internal state. * </p> * * @return Whether the file was read or not. * @throws IOException * if there is an error reading the file. * @throws InterruptedException * if there is an error. */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!isProcessed) { if (mFileLength > (long) Integer.MAX_VALUE) { throw new IOException("File is longer than Integer.MAX_VALUE."); } byte[] contents = new byte[(int) mFileLength]; FileSystem fs = mFileToRead.getFileSystem(mConf); FSDataInputStream in = null; try { // Set the contents of this file. in = fs.open(mFileToRead); IOUtils.readFully(in, contents, 0, contents.length); fileContent.set(contents, 0, contents.length); } finally { IOUtils.closeStream(in); } isProcessed = true; return true; } return false; }
From source file:com.marcolotz.lung.io.inputFormat.WholeFileRecordReader.java
License:Creative Commons License
@Override public boolean nextKeyValue() throws IOException, InterruptedException { /* if that record reader for that input split was not called yet */ if (!processed) { byte[] contents = new byte[(int) fileSplit.getLength()]; Path file = fileSplit.getPath(); // Reads from the conf file what is the desired file system. FileSystem fs = file.getFileSystem(conf); FSDataInputStream in = null;//from ww w. ja v a 2 s .com try { in = fs.open(file); IOUtils.readFully(in, contents, 0, contents.length); value.set(contents, 0, contents.length); } finally { IOUtils.closeStream(in); } processed = true; return true; } return false; }