List of usage examples for org.apache.hadoop.fs FSDataOutputStream FSDataOutputStream
public FSDataOutputStream(OutputStream out, FileSystem.Statistics stats)
From source file:com.splicemachine.fs.s3.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if ((!overwrite) && exists(path)) { throw new IOException("File already exists:" + path); }/*w w w.j a v a 2 s .c o m*/ if (!stagingDirectory.exists()) { createDirectories(stagingDirectory.toPath()); } if (!stagingDirectory.isDirectory()) { throw new IOException("Configured staging path is not a directory: " + stagingDirectory); } File tempFile = createTempFile(stagingDirectory.toPath(), "presto-s3-", ".tmp").toFile(); String key = keyFromPath(qualifiedPath(path)); return new FSDataOutputStream(new PrestoS3OutputStream(s3, transferConfig, uri.getHost(), key, tempFile, sseEnabled, sseType, sseKmsKeyId), statistics); }
From source file:com.yahoo.glimmer.indexing.preprocessor.ResourceRecordWriterTest.java
License:Open Source License
@Test public void writeSubjectAndObjectTest() throws IOException, InterruptedException, ClassNotFoundException { ByteArrayOutputStream bySubjectBos = new ByteArrayOutputStream(1024); FSDataOutputStream bySubjectOs = new FSDataOutputStream(bySubjectBos, null); ByteArrayOutputStream bySubjectOffsetsBos = new ByteArrayOutputStream(1024); FSDataOutputStream bySubjectOffsetsOs = new FSDataOutputStream(bySubjectOffsetsBos, null); e.one(fs).create(e.with(new Path(tempDirPath, "bySubject.bz2")), e.with(false)); e.will(Expectations.returnValue(bySubjectOs)); e.one(fs).create(e.with(new Path(tempDirPath, "bySubject.blockOffsets")), e.with(false)); e.will(Expectations.returnValue(bySubjectOffsetsOs)); e.one(allOs).write(e.with(new ByteMatcher("http://a/key1\nhttp://a/key2\nhttp://a/key3\n", true)), e.with(0), e.with(42));// w ww . j a v a2s.com e.one(contextOs).write(e.with(new ByteMatcher("http://a/key\n", true)), e.with(0), e.with(13)); e.one(objectOs).write(e.with(new ByteMatcher("http://a/key\nbNode123\n", true)), e.with(0), e.with(22)); e.one(predicateOs).write(e.with(new ByteMatcher("3\thttp://a/key\n", true)), e.with(0), e.with(15)); e.one(subjectOs).write(e.with(new ByteMatcher("http://a/key\n", true)), e.with(0), e.with(13)); context.checking(e); ResourceRecordWriter writer = new ResourceRecordWriter(fs, tempDirPath, null); OutputCount outputCount = new OutputCount(); outputCount.output = OUTPUT.PREDICATE; outputCount.count = 3; writer.write(new Text("http://a/key"), outputCount); outputCount.output = OUTPUT.OBJECT; outputCount.count = 0; writer.write(new Text("http://a/key"), outputCount); outputCount.output = OUTPUT.CONTEXT; outputCount.count = 0; writer.write(new Text("http://a/key"), outputCount); outputCount.output = OUTPUT.ALL; outputCount.count = 0; writer.write(new Text("http://a/key1"), outputCount); writer.write(new Text("http://a/key2"), outputCount); writer.write(new Text("http://a/key3"), outputCount); BySubjectRecord record = new BySubjectRecord(); record.setId(66); record.setPreviousId(55); record.setSubject("http://a/key"); record.addRelation("<http://predicate/> <http://Object> ."); writer.write(new Text("http://a/key"), record); outputCount.output = OUTPUT.OBJECT; outputCount.count = 0; writer.write(new Text("bNode123"), outputCount); writer.close(null); context.assertIsSatisfied(); BlockCompressedDocumentCollection collection = new BlockCompressedDocumentCollection("foo", null, 10); InputStream blockOffsetsInputStream = new ByteArrayInputStream(bySubjectOffsetsBos.toByteArray()); File bySubjectTempFile = File.createTempFile(ResourceRecordWriterTest.class.getSimpleName(), "tmp"); FileOutputStream tempFileOutputStream = new FileOutputStream(bySubjectTempFile); bySubjectBos.writeTo(tempFileOutputStream); tempFileOutputStream.flush(); tempFileOutputStream.close(); FileInputStream bySubjectFileInputStream = new FileInputStream(bySubjectTempFile); collection.init(bySubjectFileInputStream.getChannel(), blockOffsetsInputStream, 100000); blockOffsetsInputStream.close(); // Size of collection. This is the same as the number of lines written to ALL. assertEquals(3l, collection.size()); InputStream documentInputStream = collection.stream(65l); assertEquals(-1, documentInputStream.read()); documentInputStream = collection.stream(67l); assertEquals(-1, documentInputStream.read()); documentInputStream = collection.stream(66l); assertNotNull(documentInputStream); collection.close(); bySubjectFileInputStream.close(); }
From source file:com.yahoo.glimmer.indexing.preprocessor.ResourceRecordWriterTest.java
License:Open Source License
@Test public void bySubjectsTest() throws IOException, InterruptedException, NoSuchAlgorithmException, BySubjectRecordException { FSDataOutputStream bySubjectOs = new FSDataOutputStream( new FileOutputStream(new File(tempDirPath.toUri().getPath(), "bySubject.bz2")), null); FSDataOutputStream bySubjectOffsetsOs = new FSDataOutputStream( new FileOutputStream(new File(tempDirPath.toUri().getPath(), "bySubject.blockOffsets")), null); e.one(fs).create(e.with(new Path(tempDirPath, "bySubject.bz2")), e.with(false)); e.will(Expectations.returnValue(bySubjectOs)); e.one(fs).create(e.with(new Path(tempDirPath, "bySubject.blockOffsets")), e.with(false)); e.will(Expectations.returnValue(bySubjectOffsetsOs)); e.allowing(subjectOs).write(e.with(new ByteMatcher()), e.with(0), e.with(Expectations.any(Integer.class))); e.allowing(allOs).write(e.with(new ByteMatcher("all\nall\n", true)), e.with(0), e.with(Expectations.any(Integer.class))); context.checking(e);/* w ww. ja va 2 s. co m*/ System.out.println("tempDirPath:" + tempDirPath); ResourceRecordWriter writer = new ResourceRecordWriter(fs, tempDirPath, null); BySubjectRecord record = new BySubjectRecord(); Random random = new Random(); for (long l = 100000; l < 200000; l += (random.nextInt(19) + 2)) { record.setId(l); record.setSubject("Subject:" + Integer.toString(random.nextInt())); for (int i = 0; i < random.nextInt() % 100; i++) { record.addRelation("a relation " + Long.toString(random.nextLong())); } writer.write(null, record); record.setPreviousId(l); record.clearRelations(); } BySubjectRecord beforeBigRecord = new BySubjectRecord(); beforeBigRecord.setId(200200l); beforeBigRecord.setPreviousId(record.getId()); beforeBigRecord.setSubject("Before Big Test Record"); writer.write(null, beforeBigRecord); // Write a big record that will span multiple blocks of 100000 bytes. BySubjectRecord bigRecord = new BySubjectRecord(); bigRecord.setId(200201l); bigRecord.setPreviousId(beforeBigRecord.getId()); bigRecord.setSubject("Big Test Record"); MessageDigest md5Digest = MessageDigest.getInstance("MD5"); StringBuilder sb = new StringBuilder(); // 8k x 128 byte relations. The relation here is just a 128 byte hex string without delimiters. for (int i = 0; i < 8192; i++) { md5Digest.update((byte) ((i * 1299299) & 0xFF)); byte[] digest = md5Digest.digest(); sb.append(Hex.encodeHex(digest)); md5Digest.update(digest); digest = md5Digest.digest(); sb.append(Hex.encodeHex(digest)); md5Digest.update(digest); digest = md5Digest.digest(); sb.append(Hex.encodeHex(digest)); md5Digest.update(digest); digest = md5Digest.digest(); sb.append(Hex.encodeHex(digest)); bigRecord.addRelation(sb.toString()); sb.setLength(0); } writer.write(null, bigRecord); BySubjectRecord afterBigRecord = new BySubjectRecord(); afterBigRecord.setId(200202l); afterBigRecord.setPreviousId(bigRecord.getId()); afterBigRecord.setSubject("After Big Test Record"); writer.write(null, afterBigRecord); OutputCount outputCount = new OutputCount(); outputCount.output = OUTPUT.ALL; outputCount.count = 1; Text key = new Text("all"); for (int i = 0; i < 200205; i++) { writer.write(key, outputCount); } writer.write(new Text("http://a/key1"), outputCount); writer.close(null); BlockCompressedDocumentCollection collection = new BlockCompressedDocumentCollection("bySubject", null, 10); String indexBaseName = new File(tempDirPath.toUri().getPath(), "bySubject").getCanonicalPath(); collection.filename(indexBaseName); assertEquals(-1, collection.stream(99999).read()); InputStream documentInputStream = collection.stream(100000); record.readFrom(new InputStreamReader(documentInputStream)); assertEquals(100000, record.getId()); documentInputStream = collection.stream(record.getId()); record.readFrom(new InputStreamReader(documentInputStream)); assertEquals(record.getId(), record.getId()); record.setPreviousId(3); record.setSubject(null); documentInputStream = collection.stream(record.getId() + 1); assertEquals(-1, documentInputStream.read()); documentInputStream = collection.stream(beforeBigRecord.getId()); record.readFrom(new InputStreamReader(documentInputStream)); assertEquals(beforeBigRecord, record); documentInputStream = collection.stream(afterBigRecord.getId()); record.readFrom(new InputStreamReader(documentInputStream)); assertEquals(afterBigRecord, record); documentInputStream = collection.stream(bigRecord.getId()); record.readFrom(new InputStreamReader(documentInputStream)); System.out.println("BigRecord Relation count:" + bigRecord.getRelationsCount()); System.out.println("First:" + bigRecord.getRelation(0)); System.out.println("Last:" + bigRecord.getRelation(bigRecord.getRelationsCount() - 1)); System.out.println("Record Relation count:" + record.getRelationsCount()); System.out.println("First:" + record.getRelation(0)); System.out.println("Last:" + record.getRelation(record.getRelationsCount() - 1)); int limit = bigRecord.getRelationsCount() > record.getRelationsCount() ? record.getRelationsCount() : bigRecord.getRelationsCount(); for (int i = 0; i < limit; i++) { assertEquals("At index " + i, bigRecord.getRelation(i), record.getRelation(i)); } assertEquals(bigRecord.getRelationsCount(), record.getRelationsCount()); assertEquals(bigRecord, record); assertEquals(-1, collection.stream(afterBigRecord.getId() + 1).read()); collection.close(); }
From source file:com.yahoo.glimmer.util.ComputeHashToolTest.java
License:Open Source License
@Before public void before() throws IOException { context = new Mockery(); context.setImposteriser(ClassImposteriser.INSTANCE); fs = context.mock(FileSystem.class); final FileStatus inPathStatus = context.mock(FileStatus.class, "inPathStatus"); final Path inPath = new Path("filename"); unsignedPath = new PathMatcher("filename.map"); signedPath = new PathMatcher("filename.smap"); infoPath = new PathMatcher("filename.mapinfo"); unsignedStream = new ByteArrayOutputStream(4096); signedStream = new ByteArrayOutputStream(4096); infoStream = new ByteArrayOutputStream(4096); expectations = new Expectations() { {//from w w w . ja va 2 s . c om one(fs).getFileStatus(with(inPath)); will(returnValue(inPathStatus)); one(inPathStatus).isDirectory(); will(returnValue(false)); one(fs).getConf(); will(returnValue(new Configuration())); allowing(inPathStatus).getPath(); will(returnValue(inPath)); allowing(fs).open(with(inPath)); will(new Action() { @Override public void describeTo(Description arg0) { } @Override public Object invoke(Invocation invocation) throws Throwable { ByteArrayInputStream in = new SeekablePositionedReadableByteArrayInputStream( SOME_LINES.getBytes()); return new FSDataInputStream(in); } }); allowing(fs).create(with(infoPath), with(true)); will(returnValue(new FSDataOutputStream(infoStream, new Statistics("infoStats")))); allowing(fs).setPermission(with(infoPath), with(ComputeHashTool.ALL_PERMISSIONS)); } }; computeMph = new ComputeHashTool(); }
From source file:com.yahoo.glimmer.util.ComputeHashToolTest.java
License:Open Source License
@Test public void unsignedTest() throws IOException, ClassNotFoundException { expectations.oneOf(fs).create(expectations.with(unsignedPath), expectations.with(true)); expectations/*w w w . ja v a2 s . co m*/ .will(Expectations.returnValue(new FSDataOutputStream(unsignedStream, new Statistics("outStats")))); expectations.oneOf(fs).setPermission(expectations.with(unsignedPath), expectations.with(ComputeHashTool.ALL_PERMISSIONS)); context.checking(expectations); long hashSize = computeMph.buildHash(fs, "filename", null, true, false, Charset.forName("UTF-8"), false); assertEquals(4, hashSize); context.assertIsSatisfied(); assertEquals("", infoStream.toString()); // unmarshal the hash and check the values.. ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(unsignedStream.toByteArray())); Object readObject = ois.readObject(); assertTrue(readObject instanceof HollowTrieMonotoneMinimalPerfectHashFunction); @SuppressWarnings("unchecked") HollowTrieMonotoneMinimalPerfectHashFunction<MutableString> mph = (HollowTrieMonotoneMinimalPerfectHashFunction<MutableString>) readObject; assertEquals(0, mph.getLong("a")); assertEquals(1, mph.getLong("b")); assertEquals(2, mph.getLong("cc")); assertEquals(3, mph.getLong("d")); }
From source file:com.yahoo.glimmer.util.ComputeHashToolTest.java
License:Open Source License
@Test public void signedAndUnsignedTest() throws IOException, ClassNotFoundException { expectations.oneOf(fs).create(expectations.with(unsignedPath), expectations.with(true)); expectations.will(Expectations/* w w w . j a va2s .c o m*/ .returnValue(new FSDataOutputStream(unsignedStream, new Statistics("unsignedOutStats")))); expectations.oneOf(fs).setPermission(expectations.with(unsignedPath), expectations.with(ComputeHashTool.ALL_PERMISSIONS)); expectations.oneOf(fs).create(expectations.with(signedPath), expectations.with(true)); expectations.will( Expectations.returnValue(new FSDataOutputStream(signedStream, new Statistics("signedOutStats")))); expectations.oneOf(fs).setPermission(expectations.with(signedPath), expectations.with(ComputeHashTool.ALL_PERMISSIONS)); context.checking(expectations); long hashSize = computeMph.buildHash(fs, "filename", null, true, true, Charset.forName("UTF-8"), true); assertEquals(4, hashSize); context.assertIsSatisfied(); //assertEquals("size\t4\nunsignedBits\t728\nsignedWidth\t64\n", infoStream.toString()); The hash function has changed!? assertEquals("size\t4\nunsignedBits\t537\nsignedWidth\t64\n", infoStream.toString()); // unmarshal the hash and check the values.. ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(unsignedStream.toByteArray())); Object readObject = ois.readObject(); assertTrue(readObject instanceof HollowTrieMonotoneMinimalPerfectHashFunction); @SuppressWarnings("unchecked") HollowTrieMonotoneMinimalPerfectHashFunction<MutableString> unsignedMap = (HollowTrieMonotoneMinimalPerfectHashFunction<MutableString>) readObject; ByteBuffer signedByteBuffer = ByteBuffer.wrap(signedStream.toByteArray()); LongBigList signatures = new MyByteBufferLongBigList(signedByteBuffer); LongBigListSignedStringMap signedMap = new LongBigListSignedStringMap(unsignedMap, signatures); assertEquals(-1, signedMap.getLong("0")); assertEquals(0, signedMap.getLong("a")); assertEquals(-1, signedMap.getLong("aa")); assertEquals(1, signedMap.getLong("b")); assertEquals(-1, signedMap.getLong("bb")); assertEquals(2, signedMap.getLong("cc")); assertEquals(-1, signedMap.getLong("ca")); assertEquals(3, signedMap.getLong("d")); assertEquals(-1, signedMap.getLong("dx")); }
From source file:de.fiz.akubra.hdfs.HDFSBlobStoreConnectionTest.java
License:Apache License
@Test public void testCreateBlob1() throws Exception { expect(mockStore.getFilesystem()).andReturn(mockFs); expect(mockStore.getId()).andReturn(mockId).times(3); expect(mockFs.exists((Path) anyObject())).andReturn(false); expect(mockFs.create((Path) anyObject())) .andReturn(new FSDataOutputStream(new ByteArrayOutputStream(20), null)); expect(mockFs.exists((Path) anyObject())).andReturn(true); replay(mockStore, mockFs);/* w w w .j a v a2s.c om*/ HDFSBlobStoreConnection connection = createTestConnection(); byte[] buf = new byte[4096]; new Random().nextBytes(buf); HDFSBlob b = (HDFSBlob) connection.getBlob(new ByteArrayInputStream(buf), 4096, null); assertNotNull(b); assertTrue(b.getConnection() == connection); assertTrue(b.exists()); }
From source file:gobblin.util.ParallelRunnerTest.java
License:Apache License
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)).thenReturn( new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); }// w w w.j a va2 s . c om Assert.assertEquals(actual.toString(), expected); }
From source file:hsyndicate.hadoop.dfs.HSyndicateDFS.java
License:Apache License
@Override public synchronized FSDataOutputStream create(Path file, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { SyndicateFSPath hpath = makeSyndicateFSPath(file); if (this.syndicateFS.exists(hpath)) { if (overwrite) { delete(file);// www . jav a 2 s . com } else { throw new IOException("File already exists: " + file); } } else { Path parent = file.getParent(); if (parent != null) { if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } } } return new FSDataOutputStream(this.syndicateFS.getFileOutputStream(hpath), this.statistics); }
From source file:io.pravega.segmentstore.storage.impl.hdfs.MockFileSystem.java
License:Open Source License
@Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { FSDataOutputStream result = new FSDataOutputStream(createInternal(f).contents, null); invokeCustomAction(this.onCreate, f); return result; }