List of usage examples for org.apache.hadoop.fs FSDataInputStream FSDataInputStream
public FSDataInputStream(InputStream in)
From source file:com.ripariandata.timberwolf.writer.hive.SequenceFileMailWriterTest.java
License:Apache License
@SuppressWarnings("deprecation") private FileSystem mockFileSystem(final String path, final byte[] data) throws IOException { FileSystem fs = mock(FileSystem.class); Path fsPath = new Path(path); when(fs.open(eq(fsPath), any(int.class))) .thenReturn(new FSDataInputStream(new SeekablePositionedReadableByteArrayInputStream(data))); when(fs.getLength(eq(fsPath))).thenReturn((long) data.length); return fs;/*from w ww . ja va 2 s . c o m*/ }
From source file:com.splicemachine.fs.s3.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return new FSDataInputStream(new BufferedFSInputStream( new PrestoS3InputStream(s3, uri.getHost(), path, maxAttempts, maxBackoffTime, maxRetryTime), bufferSize));//from ww w. jav a 2 s . co m }
From source file:com.thinkbiganalytics.spark.io.ZipStreamingOutputTest.java
License:Apache License
/** * Verify streaming output.//from ww w. ja v a 2s. c om */ @Test public void test() throws Exception { // Mock file system final FileSystem fs = Mockito.mock(FileSystem.class); final Path source = new Path("/tmp/source"); final LocatedFileStatus file1 = createFile("_SUCCESS", source); final LocatedFileStatus file2 = createFile("part-0", source); Mockito.when(fs.listFiles(source, false)) .thenReturn(new ForwardingRemoteIterator<>(Iterators.forArray(file1, file2))); final FSDataInputStream fileStream = new FSDataInputStream(new SeekableNullInputStream()); Mockito.when(fs.open(file1.getPath())).thenReturn(fileStream); Mockito.when(fs.open(file2.getPath())).thenReturn(fileStream); final CountDownLatch deleteLatch = new CountDownLatch(1); Mockito.when(fs.delete(source, true)).then(new Answer<Boolean>() { @Override public Boolean answer(final InvocationOnMock invocation) { deleteLatch.countDown(); return true; } }); // Write ZIP to output stream final ByteArrayOutputStream out = new ByteArrayOutputStream(); final ZipStreamingOutput zip = new ZipStreamingOutput(source, fs); zip.write(out); // Verify output stream final ZipInputStream in = new ZipInputStream(out.toInputStream()); ZipEntry entry = in.getNextEntry(); Assert.assertNotNull("Missing _SUCCESS entry", entry); Assert.assertEquals("_SUCCESS", entry.getName()); entry = in.getNextEntry(); Assert.assertNotNull("Missing part-0 entry", entry); Assert.assertEquals("part-0", entry.getName()); entry = in.getNextEntry(); Assert.assertNull("Unexpected entry", entry); // Verify path deleted deleteLatch.await(1, TimeUnit.SECONDS); Mockito.verify(fs).delete(source, true); }
From source file:com.twitter.hraven.etl.JobHistoryFileParserHadoop2.java
License:Apache License
/** * {@inheritDoc}/*from ww w .j a va 2s.co m*/ */ @Override public void parse(byte[] historyFileContents, JobKey jobKey) throws ProcessingException { this.jobKey = jobKey; this.jobKeyBytes = jobKeyConv.toBytes(jobKey); this.jobDetails = new JobDetails(jobKey); initializeJobDetails(); setJobId(jobKey.getJobId().getJobIdString()); try { FSDataInputStream in = new FSDataInputStream(new ByteArrayWrapper(historyFileContents)); /** first line is the version, ignore it */ String versionIgnore = in.readLine(); /** second line in file is the schema */ this.schema = schema.parse(in.readLine()); /** now figure out the schema */ understandSchema(schema.toString()); /** now read the rest of the file */ this.reader = new GenericDatumReader<GenericRecord>(schema); this.decoder = DecoderFactory.get().jsonDecoder(schema, in); GenericRecord record = null; Hadoop2RecordType recType = null; try { while ((record = reader.read(null, decoder)) != null) { if (record.get(TYPE) != null) { recType = EVENT_RECORD_NAMES.get(record.get(TYPE).toString()); } else { throw new ProcessingException("expected one of " + Arrays.asList(Hadoop2RecordType.values()) + " \n but not found, cannot process this record! " + jobKey); } if (recType == null) { throw new ProcessingException("new record type has surfaced: " + record.get(TYPE).toString() + " cannot process this record! " + jobKey); } // GenericRecord's get returns an Object Object eDetails = record.get(EVENT); // confirm that we got an "event" object if (eDetails != null) { JSONObject eventDetails = new JSONObject(eDetails.toString()); processRecords(recType, eventDetails); } else { throw new ProcessingException("expected event details but not found " + record.get(TYPE).toString() + " cannot process this record! " + jobKey); } } } catch (EOFException eof) { // not an error, simply end of file LOG.info("Done parsing file, reached eof for " + jobKey); } } catch (IOException ioe) { throw new ProcessingException(" Unable to parse history file in function parse, " + "cannot process this record!" + jobKey + " error: ", ioe); } catch (JSONException jse) { throw new ProcessingException(" Unable to parse history file in function parse, " + "cannot process this record! " + jobKey + " error: ", jse); } catch (IllegalArgumentException iae) { throw new ProcessingException(" Unable to parse history file in function parse, " + "cannot process this record! " + jobKey + " error: ", iae); } /* * set the job status for this job once the entire file is parsed * this has to be done separately * since JOB_FINISHED event is missing the field jobStatus, * where as JOB_KILLED and JOB_FAILED * events are not so we need to look through the whole file to confirm * the job status and then generate the put */ Put jobStatusPut = getJobStatusPut(); this.jobPuts.add(jobStatusPut); // set the hadoop version for this record Put versionPut = getHadoopVersionPut(JobHistoryFileParserFactory.getHistoryFileVersion2(), this.jobKeyBytes); this.jobPuts.add(versionPut); LOG.info("For " + this.jobKey + " #jobPuts " + jobPuts.size() + " #taskPuts: " + taskPuts.size()); }
From source file:com.uber.hoodie.common.table.log.HoodieLogFileReader.java
License:Apache License
HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean readBlockLazily, boolean reverseReader) throws IOException { FSDataInputStream fsDataInputStream = fs.open(logFile.getPath(), bufferSize); if (fsDataInputStream.getWrappedStream() instanceof FSInputStream) { this.inputStream = new FSDataInputStream( new BufferedFSInputStream((FSInputStream) fsDataInputStream.getWrappedStream(), bufferSize)); } else {/*w ww . j a va2 s.c o m*/ // fsDataInputStream.getWrappedStream() maybe a BufferedFSInputStream // need to wrap in another BufferedFSInputStream the make bufferSize work? this.inputStream = fsDataInputStream; } this.logFile = logFile; this.readerSchema = readerSchema; this.readBlockLazily = readBlockLazily; this.reverseReader = reverseReader; if (this.reverseReader) { this.reverseLogFilePosition = this.lastReverseLogFilePosition = fs.getFileStatus(logFile.getPath()) .getLen(); } addShutDownHook(); }
From source file:com.yahoo.glimmer.util.ComputeHashToolTest.java
License:Open Source License
@Before public void before() throws IOException { context = new Mockery(); context.setImposteriser(ClassImposteriser.INSTANCE); fs = context.mock(FileSystem.class); final FileStatus inPathStatus = context.mock(FileStatus.class, "inPathStatus"); final Path inPath = new Path("filename"); unsignedPath = new PathMatcher("filename.map"); signedPath = new PathMatcher("filename.smap"); infoPath = new PathMatcher("filename.mapinfo"); unsignedStream = new ByteArrayOutputStream(4096); signedStream = new ByteArrayOutputStream(4096); infoStream = new ByteArrayOutputStream(4096); expectations = new Expectations() { {/*from ww w .ja v a2 s . c o m*/ one(fs).getFileStatus(with(inPath)); will(returnValue(inPathStatus)); one(inPathStatus).isDirectory(); will(returnValue(false)); one(fs).getConf(); will(returnValue(new Configuration())); allowing(inPathStatus).getPath(); will(returnValue(inPath)); allowing(fs).open(with(inPath)); will(new Action() { @Override public void describeTo(Description arg0) { } @Override public Object invoke(Invocation invocation) throws Throwable { ByteArrayInputStream in = new SeekablePositionedReadableByteArrayInputStream( SOME_LINES.getBytes()); return new FSDataInputStream(in); } }); allowing(fs).create(with(infoPath), with(true)); will(returnValue(new FSDataOutputStream(infoStream, new Statistics("infoStats")))); allowing(fs).setPermission(with(infoPath), with(ComputeHashTool.ALL_PERMISSIONS)); } }; computeMph = new ComputeHashTool(); }
From source file:de.fiz.akubra.hdfs.HDFSBlobTest.java
License:Apache License
@Test public void testOpenInputStream() throws Exception { expect(mockConnection.getBlobStore()).andReturn(mockStore).times(3); expect(mockConnection.isClosed()).andReturn(false).times(2); expect(mockConnection.getFileSystem()).andReturn(mockFs).times(2); expect(mockStore.getId()).andReturn(blobStoreUri).times(2); byte[] buf = new byte[1024]; new Random().nextBytes(buf); expect(mockFs.open((Path) anyObject())).andReturn(new FSDataInputStream(new SeekableInputStream(buf))); replay(mockConnection, mockFs, mockStore); HDFSBlob b = new HDFSBlob(blobUri, mockConnection); assertNotNull(b.openInputStream());// w ww . j av a2s. c o m }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FSDataInputStream open(Path f) throws IOException { long startTime = System.nanoTime(); Path unwrappedPath = unwrapPath(f); FSDataInputStream stream;/*ww w . ja v a2 s . co m*/ if (!this.skipRead) { stream = new FSDataInputStream(new WrappedFSDataInputStream(this.wrappedFS.open(unwrappedPath), f, LiveOperationStatisticsAggregator.instance, this.skipOther)); } else { stream = this.wrappedFS.open(unwrappedPath); } if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return stream; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { long startTime = System.nanoTime(); Path unwrappedPath = unwrapPath(f); FSDataInputStream stream;// w w w . j a v a 2 s .com if (!this.skipRead) { stream = new FSDataInputStream( new WrappedFSDataInputStream(this.wrappedFS.open(unwrappedPath, bufferSize), f, LiveOperationStatisticsAggregator.instance, this.skipOther)); } else { stream = this.wrappedFS.open(unwrappedPath, bufferSize); } if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return stream; }
From source file:edu.umn.cs.spatialHadoop.core.RTree.java
License:Open Source License
@Override public void readFields(DataInput in) throws IOException { // Tree size (Header + structure + data) treeSize = in.readInt();/*from w w w. j a v a2 s . co m*/ if (treeSize == 0) { height = elementCount = 0; return; } // Read only the tree structure in memory while actual records remain on // disk and loaded when necessary height = in.readInt(); if (height == 0) return; degree = in.readInt(); elementCount = in.readInt(); // Keep only tree structure in memory nodeCount = (int) ((powInt(degree, height) - 1) / (degree - 1)); int structureSize = nodeCount * NodeSize; byte[] treeStructure = new byte[structureSize]; in.readFully(treeStructure, 0, structureSize); structure = new FSDataInputStream(new MemoryInputStream(treeStructure)); if (in instanceof FSDataInputStream) { this.treeStartOffset = ((FSDataInputStream) in).getPos() - structureSize - TreeHeaderSize; this.data = (FSDataInputStream) in; } else { // Load all tree data in memory this.treeStartOffset = 0 - structureSize - TreeHeaderSize; int treeDataSize = treeSize - TreeHeaderSize - structureSize; byte[] treeData = new byte[treeDataSize]; in.readFully(treeData, 0, treeDataSize); this.data = new FSDataInputStream(new MemoryInputStream(treeData)); } nodeCount = (int) ((Math.pow(degree, height) - 1) / (degree - 1)); leafNodeCount = (int) Math.pow(degree, height - 1); nonLeafNodeCount = nodeCount - leafNodeCount; }