List of usage examples for org.apache.hadoop.fs FSDataInputStream seek
@Override public void seek(long desired) throws IOException
From source file:nl.surfsara.warcutils.WarcRecordReader.java
License:Apache License
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException { FileSplit split = (FileSplit) inputSplit; Configuration conf = context.getConfiguration(); final Path file = split.getPath(); start = split.getStart();//ww w . j a v a 2 s.c om end = start + split.getLength(); compressionCodecs = new CompressionCodecFactory(conf); codec = compressionCodecs.getCodec(file); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(split.getPath()); if (isCompressedInput()) { in = new DataInputStream(codec.createInputStream(fileIn, decompressor)); filePosition = fileIn; } else { fileIn.seek(start); in = fileIn; filePosition = fileIn; } warcReader = WarcReaderFactory.getReaderUncompressed(in); warcReader.setWarcTargetUriProfile(WarcIOConstants.URIPROFILE); warcReader.setBlockDigestEnabled(WarcIOConstants.BLOCKDIGESTENABLED); warcReader.setPayloadDigestEnabled(WarcIOConstants.PAYLOADDIGESTENABLED); warcReader.setRecordHeaderMaxSize(WarcIOConstants.HEADERMAXSIZE); warcReader.setPayloadHeaderMaxSize(WarcIOConstants.PAYLOADHEADERMAXSIZE); this.pos = start; }
From source file:org.apache.accumulo.core.file.rfile.bcfile.BoundedRangeFileInputStream.java
License:Apache License
@Override public int read(final byte[] b, final int off, int len) throws IOException { if ((off | len | (off + len) | (b.length - (off + len))) < 0) { throw new IndexOutOfBoundsException(); }/*w w w. jav a 2 s. c o m*/ final int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos))); if (n == 0) return -1; Integer ret = 0; final FSDataInputStream inLocal = in; synchronized (inLocal) { inLocal.seek(pos); try { ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws IOException { int ret = 0; ret = inLocal.read(b, off, n); return ret; } }); } catch (PrivilegedActionException e) { throw (IOException) e.getException(); } } if (ret < 0) { end = pos; return -1; } pos += ret; return ret; }
From source file:org.apache.accumulo.server.tabletserver.log.DfsLogger.java
License:Apache License
public static FSDataInputStream readHeader(VolumeManager fs, Path path, Map<String, String> opts) throws IOException { FSDataInputStream file = fs.open(path); try {//from ww w . j a va 2 s .co m byte[] magic = LOG_FILE_HEADER_V2.getBytes(); byte[] buffer = new byte[magic.length]; file.readFully(buffer); if (Arrays.equals(buffer, magic)) { int count = file.readInt(); for (int i = 0; i < count; i++) { String key = file.readUTF(); String value = file.readUTF(); opts.put(key, value); } } else { file.seek(0); return file; } return file; } catch (IOException ex) { file.seek(0); return file; } }
From source file:org.apache.accumulo.tserver.log.DfsLogger.java
License:Apache License
public static DFSLoggerInputStreams readHeaderAndReturnStream(VolumeManager fs, Path path, AccumuloConfiguration conf) throws IOException { FSDataInputStream input = fs.open(path); DataInputStream decryptingInput = null; byte[] magic = DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8); byte[] magicBuffer = new byte[magic.length]; try {/* ww w.j a va 2s .com*/ input.readFully(magicBuffer); if (Arrays.equals(magicBuffer, magic)) { // additional parameters it needs from the underlying stream. String cryptoModuleClassname = input.readUTF(); CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoModuleClassname); // Create the parameters and set the input stream into those parameters CryptoModuleParameters params = CryptoModuleFactory .createParamsObjectFromAccumuloConfiguration(conf); params.setEncryptedInputStream(input); // Create the plaintext input stream from the encrypted one params = cryptoModule.getDecryptingInputStream(params); if (params.getPlaintextInputStream() instanceof DataInputStream) { decryptingInput = (DataInputStream) params.getPlaintextInputStream(); } else { decryptingInput = new DataInputStream(params.getPlaintextInputStream()); } } else { input.seek(0); byte[] magicV2 = DfsLogger.LOG_FILE_HEADER_V2.getBytes(UTF_8); byte[] magicBufferV2 = new byte[magicV2.length]; input.readFully(magicBufferV2); if (Arrays.equals(magicBufferV2, magicV2)) { // Log files from 1.5 dump their options in raw to the logger files. Since we don't know the class // that needs to read those files, we can make a couple of basic assumptions. Either it's going to be // the NullCryptoModule (no crypto) or the DefaultCryptoModule. // If it's null, we won't have any parameters whatsoever. First, let's attempt to read // parameters Map<String, String> opts = new HashMap<>(); int count = input.readInt(); for (int i = 0; i < count; i++) { String key = input.readUTF(); String value = input.readUTF(); opts.put(key, value); } if (opts.size() == 0) { // NullCryptoModule, we're done decryptingInput = input; } else { // The DefaultCryptoModule will want to read the parameters from the underlying file, so we will put the file back to that spot. org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory .getCryptoModule(DefaultCryptoModule.class.getName()); CryptoModuleParameters params = CryptoModuleFactory .createParamsObjectFromAccumuloConfiguration(conf); // go back to the beginning, but skip over magicV2 already checked earlier input.seek(magicV2.length); params.setEncryptedInputStream(input); params = cryptoModule.getDecryptingInputStream(params); if (params.getPlaintextInputStream() instanceof DataInputStream) { decryptingInput = (DataInputStream) params.getPlaintextInputStream(); } else { decryptingInput = new DataInputStream(params.getPlaintextInputStream()); } } } else { input.seek(0); decryptingInput = input; } } } catch (EOFException e) { log.warn("Got EOFException trying to read WAL header information, assuming the rest of the file (" + path + ") has no data."); // A TabletServer might have died before the (complete) header was written throw new LogHeaderIncompleteException(e); } return new DFSLoggerInputStreams(input, decryptingInput); }
From source file:org.apache.ambari.view.hive.utils.FilePaginator.java
License:Apache License
/** * Read one page of size PAGE_SIZE//from www . j ava 2 s .c o m * @param page page index * @return data in UTF-8 * @throws java.io.IOException * @throws InterruptedException */ public String readPage(long page) throws IOException, InterruptedException { FSDataInputStream stream = hdfsApi.open(filePath); try { stream.seek(page * PAGE_SIZE); } catch (IOException e) { throw new IllegalArgumentException("Page " + page + " does not exists"); } byte[] buffer = new byte[PAGE_SIZE]; int readCount = 0; int read = 0; while (read < PAGE_SIZE) { try { readCount = stream.read(buffer, read, PAGE_SIZE - read); } catch (IOException e) { stream.close(); throw e; } if (readCount == -1) break; read += readCount; } if (read != 0) { byte[] readData = Arrays.copyOfRange(buffer, 0, read); return new String(readData, Charset.forName("UTF-8")); } else { if (page == 0) { return ""; } throw new IllegalArgumentException("Page " + page + " does not exists"); } }
From source file:org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl.java
License:Apache License
/** * This method will be used to read from file based on number of bytes to be read and positon * * @param channel file channel/* ww w .j av a 2 s.co m*/ * @param size number of bytes * @param offset position * @return byte buffer */ private byte[] read(FSDataInputStream channel, int size, long offset) { byte[] byteBffer = new byte[size]; try { channel.seek(offset); channel.readFully(byteBffer); } catch (Exception e) { LOGGER.error(e, e.getMessage()); } return byteBffer; }
From source file:org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl.java
License:Apache License
@Override public int readInt(String filePath, long offset) { FSDataInputStream fileChannel = updateCache(filePath); int i = -1;// w ww .j a v a 2s.c o m try { fileChannel.seek(offset); i = fileChannel.readInt(); } catch (IOException e) { LOGGER.error(e, e.getMessage()); } return i; }
From source file:org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl.java
License:Apache License
@Override public long readDouble(String filePath, long offset) { FSDataInputStream fileChannel = updateCache(filePath); long i = -1;/*from w w w .ja v a2 s . c o m*/ try { fileChannel.seek(offset); i = fileChannel.readLong(); } catch (IOException e) { LOGGER.error(e, e.getMessage()); } return i; }
From source file:org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl.java
License:Apache License
@Override public long readLong(String filePath, long offset) { FSDataInputStream fileChannel = updateCache(filePath); long i = -1;/*from ww w . j av a 2s. c o m*/ try { fileChannel.seek(offset); i = fileChannel.readLong(); } catch (IOException e) { LOGGER.error(e, e.getMessage()); } return i; }
From source file:org.apache.carbondata.core.datastorage.store.impl.FileFactory.java
License:Apache License
/** * return the datainputStream which is seek to the offset of file * * @param path/*w w w. java 2 s . c o m*/ * @param fileType * @param bufferSize * @param offset * @return DataInputStream * @throws IOException */ public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize, long offset) throws IOException { path = path.replace("\\", "/"); switch (fileType) { case HDFS: case VIEWFS: Path pt = new Path(path); FileSystem fs = FileSystem.get(configuration); FSDataInputStream stream = fs.open(pt, bufferSize); stream.seek(offset); return new DataInputStream(new BufferedInputStream(stream)); default: FileInputStream fis = new FileInputStream(path); long actualSkipSize = 0; long skipSize = offset; while (actualSkipSize != offset) { actualSkipSize += fis.skip(skipSize); skipSize = skipSize - actualSkipSize; } return new DataInputStream(new BufferedInputStream(fis)); } }