List of usage examples for org.apache.hadoop.fs FSDataInputStream FSDataInputStream
public FSDataInputStream(InputStream in)
From source file:org.apache.accumulo.core.file.rfile.MultiLevelIndexTest.java
License:Apache License
private void runTest(int maxBlockSize, int num) throws IOException { AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); FSDataOutputStream dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a")); CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(dos), "gz", CachedConfiguration.getInstance(), aconf); BufferedWriter mliw = new BufferedWriter(new Writer(_cbw, maxBlockSize)); for (int i = 0; i < num; i++) mliw.add(new Key(String.format("%05d000", i)), i, 0, 0, 0); mliw.addLast(new Key(String.format("%05d000", num)), num, 0, 0, 0); ABlockWriter root = _cbw.prepareMetaBlock("root"); mliw.close(root);//from w ww . ja v a 2s . c o m root.close(); _cbw.close(); dos.close(); baos.close(); byte[] data = baos.toByteArray(); SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data); FSDataInputStream in = new FSDataInputStream(bais); CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in, data.length, CachedConfiguration.getInstance(), aconf); Reader reader = new Reader(_cbr, RFile.RINDEX_VER_8); BlockRead rootIn = _cbr.getMetaBlock("root"); reader.readFields(rootIn); rootIn.close(); IndexIterator liter = reader.lookup(new Key("000000")); int count = 0; while (liter.hasNext()) { assertEquals(count, liter.nextIndex()); assertEquals(count, liter.peek().getNumEntries()); assertEquals(count, liter.next().getNumEntries()); count++; } assertEquals(num + 1, count); while (liter.hasPrevious()) { count--; assertEquals(count, liter.previousIndex()); assertEquals(count, liter.peekPrevious().getNumEntries()); assertEquals(count, liter.previous().getNumEntries()); } assertEquals(0, count); // go past the end liter = reader.lookup(new Key(String.format("%05d000", num + 1))); assertFalse(liter.hasNext()); Random rand = new Random(); for (int i = 0; i < 100; i++) { int k = rand.nextInt(num * 1000); int expected; if (k % 1000 == 0) expected = k / 1000; // end key is inclusive else expected = k / 1000 + 1; liter = reader.lookup(new Key(String.format("%08d", k))); IndexEntry ie = liter.next(); assertEquals(expected, ie.getNumEntries()); } }
From source file:org.apache.accumulo.core.file.rfile.RFileTest.java
License:Apache License
private void runVersionTest(int version) throws IOException { InputStream in = this.getClass().getClassLoader() .getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf"); ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte buf[] = new byte[1024]; int read;//from ww w. j ava2s . c o m while ((read = in.read(buf)) > 0) baos.write(buf, 0, read); byte data[] = baos.toByteArray(); SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data); FSDataInputStream in2 = new FSDataInputStream(bais); AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in2, data.length, CachedConfiguration.getInstance(), aconf); Reader reader = new RFile.Reader(_cbr); checkIndex(reader); ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader); for (int start : new int[] { 0, 10, 100, 998 }) { for (int cf = 1; cf <= 4; cf++) { if (start == 0) iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true); else iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true); for (int i = start; i < 1000; i++) { assertTrue(iter.hasTop()); assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey()); assertEquals(newValue(i + ""), iter.getTopValue()); iter.next(); } assertFalse(iter.hasTop()); } if (start == 0) iter.seek(new Range(), newColFamByteSequence(), false); else iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false); for (int i = start; i < 1000; i++) { for (int cf = 1; cf <= 4; cf++) { assertTrue(iter.hasTop()); assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey()); assertEquals(newValue(i + ""), iter.getTopValue()); iter.next(); } } assertFalse(iter.hasTop()); } reader.close(); }
From source file:org.apache.ambari.view.hive.utils.HdfsApiMock.java
License:Apache License
protected SeekableByteArrayInputStream setupQueryInputStream(HdfsApi hdfsApi, String query) throws IOException, InterruptedException { SeekableByteArrayInputStream inputStreamHQL = new SeekableByteArrayInputStream(query.getBytes()); expect(hdfsApi.open(endsWith(".hql"))).andReturn(new FSDataInputStream(inputStreamHQL)).anyTimes(); return inputStreamHQL; }
From source file:org.apache.avro.tool.CatTool.java
License:Apache License
private void nextInput() throws IOException { currentInput++;//from w ww. j a va 2s .c o m Path path = inFiles.get(currentInput); FSDataInputStream input = new FSDataInputStream(Util.openFromFS(path)); reader = new DataFileStream<GenericRecord>(input, new GenericDatumReader<GenericRecord>()); if (schema == null) { // if this is the first file, the schema gets saved schema = reader.getSchema(); } else if (!schema.equals(reader.getSchema())) { // subsequent files have to have equal schemas throw new IOException("schemas dont match"); } }
From source file:org.apache.cassandra.hadoop.fs.CassandraFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { INode inode = checkFile(path);/* w ww. j a v a2s .co m*/ return new FSDataInputStream(new CassandraInputStream(getConf(), store, inode, statistics)); }
From source file:org.apache.distributedlog.fs.DLFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { try {/* w w w. java 2 s. co m*/ DistributedLogManager dlm = namespace.openLog(getStreamName(path)); LogReader reader; try { reader = dlm.openLogReader(DLSN.InitialDLSN); } catch (LogNotFoundException lnfe) { throw new FileNotFoundException(path.toString()); } catch (LogEmptyException lee) { throw new FileNotFoundException(path.toString()); } return new FSDataInputStream(new BufferedFSInputStream(new DLInputStream(dlm, reader, 0L), bufferSize)); } catch (LogNotFoundException e) { throw new FileNotFoundException(path.toString()); } }
From source file:org.apache.drill.exec.store.CachedSingleFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int arg1) throws IOException { if (!path.toString().equals(this.path)) { throw new IOException( String.format("You requested file %s but this cached single file system only has the file %s.", path.toString(), this.path)); }//from w ww . ja v a2 s. c o m return new FSDataInputStream(new CachedFSDataInputStream(file.slice())); }
From source file:org.apache.drill.exec.store.ClassPathFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path arg0, int arg1) throws IOException { String file = getFileName(arg0); URL url = Resources.getResource(file); if (url == null) { throw new IOException(String.format("Unable to find path %s.", arg0.getName())); }/*from w w w . j a v a 2s. co m*/ ResourceInputStream ris = new ResourceInputStream(Resources.toByteArray(url)); return new FSDataInputStream(ris); }
From source file:org.apache.drill.exec.store.LocalSyncableFileSystem.java
License:Apache License
@Override public FSDataInputStream open(Path path, int i) throws IOException { return new FSDataInputStream(new LocalInputStream(path)); }
From source file:org.apache.flink.runtime.fs.hdfs.HadoopDataInputStreamTest.java
License:Apache License
@Test public void testSeekSkip() throws IOException { verifyInputStream = spy(new FSDataInputStream( new SeekableByteArrayInputStream(new byte[2 * HadoopDataInputStream.MIN_SKIP_BYTES]))); testInputStream = new HadoopDataInputStream(verifyInputStream); seekAndAssert(10);/* w ww. j a v a 2s . c o m*/ seekAndAssert(10 + HadoopDataInputStream.MIN_SKIP_BYTES + 1); seekAndAssert(testInputStream.getPos() - 1); seekAndAssert(testInputStream.getPos() + 1); seekAndAssert(testInputStream.getPos() - HadoopDataInputStream.MIN_SKIP_BYTES); seekAndAssert(testInputStream.getPos()); seekAndAssert(0); seekAndAssert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES); seekAndAssert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES - 1); try { seekAndAssert(-1); Assert.fail(); } catch (Exception ignore) { } try { seekAndAssert(-HadoopDataInputStream.MIN_SKIP_BYTES - 1); Assert.fail(); } catch (Exception ignore) { } }