List of usage examples for org.apache.hadoop.fs FSDataInputStream FSDataInputStream
public FSDataInputStream(InputStream in)
From source file:gobblin.source.extractor.extract.sftp.SftpLightWeightFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { SftpGetMonitor monitor = new SftpGetMonitor(); try {//from www . j a va 2 s. c om ChannelSftp channelSftp = this.fsHelper.getSftpChannel(); InputStream is = channelSftp.get(HadoopUtils.toUriPath(path), monitor); return new FSDataInputStream( new BufferedFSInputStream(new SftpFsHelper.SftpFsFileInputStream(is, channelSftp), bufferSize)); } catch (SftpException e) { throw new IOException(e); } }
From source file:gobblin.util.io.StreamUtils.java
License:Apache License
/** * Convert an instance of {@link InputStream} to a {@link FSDataInputStream} that is {@link Seekable} and * {@link PositionedReadable}.//from ww w . j a v a 2s.c om * * @see SeekableFSInputStream * */ public static FSDataInputStream convertStream(InputStream in) throws IOException { return new FSDataInputStream(new SeekableFSInputStream(in)); }
From source file:gobblin.util.ParallelRunnerTest.java
License:Apache License
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)).thenReturn( new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); }//from w w w . j a v a 2 s . c om Assert.assertEquals(actual.toString(), expected); }
From source file:hsyndicate.hadoop.dfs.HSyndicateDFS.java
License:Apache License
@Override public synchronized FSDataInputStream open(Path path, int bufferSize) throws IOException { SyndicateFSPath hpath = makeSyndicateFSPath(path); if (!this.syndicateFS.exists(hpath)) { throw new IOException("No such file."); }/* w w w . j ava 2 s.com*/ if (this.syndicateFS.isDirectory(hpath)) { throw new IOException("Path " + path + " is a directory."); } return new FSDataInputStream(new HSyndicateInputStream(this.syndicateFS, hpath, this.statistics)); }
From source file:hudson.gridmaven.gridlayer.HadoopInstance.java
License:Open Source License
/** * This method decompress filesystem structure from HDFS archive *///from w w w .j a va2 s . c om public void getAndUntar(String src, String targetPath) throws FileNotFoundException, IOException { BufferedOutputStream dest = null; InputStream tarArchiveStream = new FSDataInputStream(fs.open(new Path(src))); TarArchiveInputStream tis = new TarArchiveInputStream(new BufferedInputStream(tarArchiveStream)); TarArchiveEntry entry = null; try { while ((entry = tis.getNextTarEntry()) != null) { int count; File outputFile = new File(targetPath, entry.getName()); if (entry.isDirectory()) { // entry is a directory if (!outputFile.exists()) { outputFile.mkdirs(); } } else { // entry is a file byte[] data = new byte[BUFFER_MAX]; FileOutputStream fos = new FileOutputStream(outputFile); dest = new BufferedOutputStream(fos, BUFFER_MAX); while ((count = tis.read(data, 0, BUFFER_MAX)) != -1) { dest.write(data, 0, count); } dest.flush(); dest.close(); } } } catch (Exception e) { e.printStackTrace(); } finally { if (dest != null) { dest.flush(); dest.close(); } tis.close(); } }
From source file:hudson.gridmaven.MavenBuilder.java
License:Open Source License
public void getAndUntar(FileSystem fs, String src, String targetPath) throws FileNotFoundException, IOException { BufferedOutputStream dest = null; InputStream tarArchiveStream = new FSDataInputStream(fs.open(new Path(src))); TarArchiveInputStream tis = new TarArchiveInputStream(new BufferedInputStream(tarArchiveStream)); TarArchiveEntry entry = null;//w w w.ja va 2s. c o m try { while ((entry = tis.getNextTarEntry()) != null) { int count; File outputFile = new File(targetPath, entry.getName()); if (entry.isDirectory()) { // entry is a directory if (!outputFile.exists()) { outputFile.mkdirs(); } } else { // entry is a file byte[] data = new byte[BUFFER_MAX]; FileOutputStream fos = new FileOutputStream(outputFile); dest = new BufferedOutputStream(fos, BUFFER_MAX); while ((count = tis.read(data, 0, BUFFER_MAX)) != -1) { dest.write(data, 0, count); } dest.flush(); dest.close(); } } } catch (Exception e) { e.printStackTrace(); } finally { if (dest != null) { dest.flush(); dest.close(); } tis.close(); } }
From source file:io.hops.erasure_coding.ReedSolomonDecoder.java
License:Apache License
protected int[] buildInputs(FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, boolean fixSource, long errorOffset, FSDataInputStream[] inputs) throws IOException { LOG.info("Building inputs to recover block starting at " + errorOffset); try {/*w ww . j av a 2s . com*/ FileStatus srcStat = fs.getFileStatus(srcFile); FileStatus parityStat = fs.getFileStatus(parityFile); long blockSize = srcStat.getBlockSize(); long blockIdx = (int) (errorOffset / blockSize); long stripeIdx; if (fixSource) { stripeIdx = blockIdx / stripeSize; } else { stripeIdx = blockIdx / paritySize; } LOG.info("FileSize = " + srcStat.getLen() + ", blockSize = " + blockSize + ", blockIdx = " + blockIdx + ", stripeIdx = " + stripeIdx); ArrayList<Integer> erasedLocations = new ArrayList<Integer>(); // First open streams to the parity blocks. for (int i = 0; i < paritySize; i++) { long offset = blockSize * (stripeIdx * paritySize + i); if ((!fixSource) && offset == errorOffset) { LOG.info(parityFile + ":" + offset + " is known to have error, adding zeros as input " + i); inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(offset + blockSize)); erasedLocations.add(i); } else if (offset > parityStat.getLen()) { LOG.info(parityFile + ":" + offset + " is past file size, adding zeros as input " + i); inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(offset + blockSize)); } else { FSDataInputStream in = parityFs.open(parityFile, conf.getInt("io.file.buffer.size", 64 * 1024)); in.seek(offset); LOG.info("Adding " + parityFile + ":" + offset + " as input " + i); inputs[i] = in; } } // Now open streams to the data blocks. for (int i = paritySize; i < paritySize + stripeSize; i++) { long offset = blockSize * (stripeIdx * stripeSize + i - paritySize); if (fixSource && offset == errorOffset) { LOG.info(srcFile + ":" + offset + " is known to have error, adding zeros as input " + i); inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(offset + blockSize)); erasedLocations.add(i); } else if (offset > srcStat.getLen()) { LOG.info(srcFile + ":" + offset + " is past file size, adding zeros as input " + i); inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(offset + blockSize)); } else { FSDataInputStream in = fs.open(srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); in.seek(offset); LOG.info("Adding " + srcFile + ":" + offset + " as input " + i); inputs[i] = in; } } if (erasedLocations.size() > paritySize) { String msg = "Too many erased locations: " + erasedLocations.size(); LOG.error(msg); throw new IOException(msg); } int[] locs = new int[erasedLocations.size()]; for (int i = 0; i < locs.length; i++) { locs[i] = erasedLocations.get(i); } return locs; } catch (IOException e) { RaidUtils.closeStreams(inputs); throw e; } }
From source file:io.hops.erasure_coding.XORDecoder.java
License:Apache License
@Override protected long fixErasedBlockImpl(FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, boolean fixSource, long blockSize, long errorOffset, long limit, boolean partial, OutputStream out, Progressable reporter, CRC32 crc) throws IOException { if (partial) { throw new IOException("We don't support partial reconstruction"); }/*from w w w . ja v a 2s . c o m*/ LOG.info("Fixing block at " + srcFile + ":" + errorOffset + ", limit " + limit); if (crc != null) { crc.reset(); } FileStatus srcStat = fs.getFileStatus(srcFile); FSDataInputStream[] inputs = new FSDataInputStream[stripeSize + this.codec.parityLength]; try { long errorBlockOffset = (errorOffset / blockSize) * blockSize; long[] srcOffsets = stripeOffsets(errorOffset, blockSize, fixSource); for (int i = 0; i < srcOffsets.length; i++) { if (fixSource && srcOffsets[i] == errorBlockOffset) { inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(blockSize)); LOG.info("Using zeros at " + srcFile + ":" + errorBlockOffset); continue; } if (srcOffsets[i] < srcStat.getLen()) { FSDataInputStream in = fs.open(srcFile); in.seek(srcOffsets[i]); inputs[i] = in; } else { inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(blockSize)); LOG.info("Using zeros at " + srcFile + ":" + errorBlockOffset); } } if (fixSource) { FSDataInputStream parityFileIn = parityFs.open(parityFile); parityFileIn.seek(parityOffset(errorOffset, blockSize)); inputs[inputs.length - 1] = parityFileIn; } else { inputs[inputs.length - 1] = new FSDataInputStream(new RaidUtils.ZeroInputStream(blockSize)); LOG.info("Using zeros at " + parityFile + ":" + errorBlockOffset); } } catch (IOException e) { RaidUtils.closeStreams(inputs); throw e; } int boundedBufferCapacity = 1; ParallelStreamReader parallelReader = new ParallelStreamReader(reporter, inputs, bufSize, parallelism, boundedBufferCapacity, blockSize); parallelReader.start(); try { // Loop while the number of skipped + written bytes is less than the max. long written; for (written = 0; written < limit;) { ParallelStreamReader.ReadResult readResult; try { readResult = parallelReader.getReadResult(); } catch (InterruptedException e) { throw new IOException("Interrupted while waiting for read result"); } // Cannot tolerate any IO errors. IOException readEx = readResult.getException(); if (readEx != null) { throw readEx; } int toWrite = (int) Math.min((long) bufSize, limit - written); XOREncoder.xor(readResult.readBufs, writeBufs[0]); out.write(writeBufs[0], 0, toWrite); if (crc != null) { crc.update(writeBufs[0], 0, toWrite); } written += toWrite; } return written; } finally { // Inputs will be closed by parallelReader.shutdown(). parallelReader.shutdown(); } }
From source file:io.pravega.segmentstore.storage.impl.hdfs.MockFileSystem.java
License:Open Source License
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { return new FSDataInputStream(new SeekableInputStream(getFileData(f).contents.toByteArray())); }
From source file:io.prestosql.plugin.hive.s3.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) { return new FSDataInputStream(new BufferedFSInputStream( new PrestoS3InputStream(s3, getBucketName(uri), path, maxAttempts, maxBackoffTime, maxRetryTime), bufferSize));/* ww w. ja va 2s.co m*/ }