List of usage examples for org.apache.hadoop.fs FSDataInputStream FSDataInputStream
public FSDataInputStream(InputStream in)
From source file:com.aliyun.odps.fs.VolumeFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Path absF = fixRelativePart(f); String filePath = getPathName(absF); if (!exists(absF)) { throw new FileNotFoundException(filePath); }/* ww w.j a v a 2s .c o m*/ FileStatus fileStatus = getFileStatus(f); if (fileStatus.isDirectory()) { throw new FileNotFoundException(VolumeFSErrorMessageGenerator.isADirectory(filePath)); } return new FSDataInputStream( new VolumeFSInputStream(filePath, volumeClient, fileStatus.getLen(), getConf())); }
From source file:com.asakusafw.runtime.compatibility.hadoop2.SequenceFileCompatibilityHadoop2.java
License:Apache License
@Override public SequenceFile.Reader openReader(InputStream in, long length, Configuration conf) throws IOException { if (in == null) { throw new IllegalArgumentException("in must not be null"); //$NON-NLS-1$ }/*from w w w . j av a2 s.c om*/ if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ } return new SequenceFile.Reader(conf, SequenceFile.Reader.stream(new FSDataInputStream(new WrappedInputStream(in))), SequenceFile.Reader.length(length)); }
From source file:com.bigstep.datalake.DLFileSystem.java
License:Apache License
@Override public FSDataInputStream open(final Path f, final int buffersize) throws IOException { statistics.incrementReadOps(1);//from w ww .ja v a 2s . c o m final HttpOpParam.Op op = GetOpParam.Op.OPEN; if (this.shouldUseEncryption) { DLEncryptionUtils dlenc = DLEncryptionUtils.getInstance(); int nHeaderSize = dlenc.getHeaderDetailLength(); byte[] b = new byte[nHeaderSize]; byte[] key = dlenc.getSecretKey(); int nBlockSize = key.length; byte[] initVector = new byte[nBlockSize]; // use a runner so the open can recover from an invalid token FsPathConnectionRunner runnerHeader = new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize), new LengthParam((long) nHeaderSize + nBlockSize)); OffsetUrlInputStream inputStreamHeader = new OffsetUrlInputStream(new UnresolvedUrlOpener(runnerHeader), new OffsetUrlOpener(null)); inputStreamHeader.readFully(0, b, 0, nHeaderSize); inputStreamHeader.readFully(nHeaderSize, initVector, 0, nBlockSize); FsPathConnectionRunner runnerData = new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize), new OffsetParam((long) nHeaderSize + nBlockSize)); OffsetUrlInputStream inputStreamData = new OffsetUrlInputStream(new UnresolvedUrlOpener(runnerData), new OffsetUrlOpener(null)); Properties props = new Properties(); props.put("commons.crypto.stream.buffer.size", buffersize); return new FSDataInputStream(new DLPositionedCryptoInputStream(props, new DLStreamInput(inputStreamData, buffersize), key, initVector, 0L)); } else { FsPathConnectionRunner runner = new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize)); return new FSDataInputStream( new OffsetUrlInputStream(new UnresolvedUrlOpener(runner), new OffsetUrlOpener(null))); } }
From source file:com.ceph.rados.fs.hdfs.RadosFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return new FSDataInputStream(new RadosHDFSInputStream(store, path.toString())); }
From source file:com.cloudera.cdk.morphline.hadoop.rcfile.SingleStreamFileSystem.java
License:Apache License
public SingleStreamFileSystem(InputStream inputStream, Path path) throws IOException { this.inputStream = new FSDataInputStream(inputStream); this.path = path; // Since this is a stream, we dont know the length of the stream. Setting it // to the maximum size this.fileStatus = new FileStatus(Long.MAX_VALUE, false, 0, 0, 0, path); }
From source file:com.cloudera.hoop.client.fs.HoopFileSystem.java
License:Open Source License
/** * Opens an FSDataInputStream at the indicated Path. * </p>/* w w w. j av a 2 s .co m*/ * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the * <code>PositionReadable</code> and <code>Seekable</code> methods. * * @param f the file name to open * @param bufferSize the size of the buffer to be used. */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Map<String, String> params = new HashMap<String, String>(); HttpURLConnection conn = getConnection("GET", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); return new FSDataInputStream(new HoopFSDataInputStream(conn.getInputStream(), bufferSize)); }
From source file:com.epam.catgenome.util.HdfsSeekableInputStream.java
License:Open Source License
/** * @param stream a HDFS {@link InputStream} * @throws IOException//from w ww.j a va2 s.c o m */ public HdfsSeekableInputStream(InputStream stream) throws IOException { this.stream = new FSDataInputStream(stream); }
From source file:com.facebook.presto.hive.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return new FSDataInputStream(new BufferedFSInputStream( new PrestoS3InputStream(s3, uri.getHost(), path, maxClientRetries), bufferSize)); }
From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return new FSDataInputStream(new BufferedFSInputStream( new PrestoS3InputStream(s3, getBucketName(uri), path, maxAttempts, maxBackoffTime, maxRetryTime), bufferSize));/*from w w w.j a v a2s . com*/ }
From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.HoplogUtil.java
License:Apache License
public static long readCleanUpIntervalMillis(FileSystem fs, Path cleanUpIntervalPath) throws IOException { if (fs.exists(cleanUpIntervalPath)) { FSDataInputStream input = new FSDataInputStream(fs.open(cleanUpIntervalPath)); long intervalDurationMillis = input.readLong(); input.close();// w ww . j ava 2 s .c o m return intervalDurationMillis; } else { return -1l; } }