Java tutorial
/** * Licensed to Cloudera, Inc. under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. Cloudera, Inc. licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloudera.flume.handlers.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.compress.DefaultCodec; import org.junit.Test; import com.cloudera.flume.conf.Context; import com.cloudera.flume.conf.FlumeConfiguration; import com.cloudera.flume.conf.SinkFactory.SinkBuilder; import com.cloudera.flume.core.Event; import com.cloudera.flume.core.EventImpl; import com.cloudera.flume.core.EventSink; import com.cloudera.util.OSUtils; /** * This tests writing to a distributed hdfs. It assumes the namenode is * localhost. * */ public class TestDFSWrite { @Test public void testDirectWrite() throws IOException { FlumeConfiguration conf = FlumeConfiguration.get(); Path path = new Path("file:///tmp/testfile"); FileSystem hdfs = path.getFileSystem(conf); hdfs.deleteOnExit(path); String STRING = "Hello World"; // writing FSDataOutputStream dos = hdfs.create(path); dos.writeUTF(STRING); dos.close(); // reading FSDataInputStream dis = hdfs.open(path); String s = dis.readUTF(); System.out.println(s); assertEquals(STRING, s); dis.close(); hdfs.close(); } @Test public void testHDFSSequenceFileWrite() throws IOException { FlumeConfiguration conf = FlumeConfiguration.get(); Path path = new Path("file:///tmp/testfile"); FileSystem hdfs = path.getFileSystem(conf); hdfs.deleteOnExit(path); Event e = new EventImpl("EVENT".getBytes()); Writer w = SequenceFile.createWriter(hdfs, conf, path, WriteableEventKey.class, WriteableEvent.class); // writing w.append(new WriteableEventKey(e), new WriteableEvent(e)); w.close(); FileStatus stats = hdfs.getFileStatus(path); assertTrue(stats.getLen() > 0); // reading SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path, conf); WriteableEventKey k = new WriteableEventKey(); WriteableEvent evt = new WriteableEvent(); r.next(k, evt); assertEquals(evt.getTimestamp(), e.getTimestamp()); assertEquals(evt.getNanos(), e.getNanos()); assertEquals(evt.getPriority(), e.getPriority()); assertTrue(Arrays.equals(evt.getBody(), e.getBody())); hdfs.close(); } @Test public void testHDFSEventSink() throws IOException, InterruptedException { FlumeConfiguration conf = FlumeConfiguration.get(); String str = "file:///tmp/testfile"; Path path = new Path(str); DFSEventSink sink = new DFSEventSink(str); FileSystem hdfs = path.getFileSystem(conf); sink.open(); Event e = new EventImpl("EVENT".getBytes()); sink.append(e); sink.close(); FileStatus stats = hdfs.getFileStatus(path); assertTrue(stats.getLen() > 0); // // reading SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path, conf); WriteableEventKey k = new WriteableEventKey(); WriteableEvent evt = new WriteableEvent(); r.next(k, evt); assertEquals(evt.getTimestamp(), e.getTimestamp()); assertEquals(evt.getNanos(), e.getNanos()); assertEquals(evt.getPriority(), e.getPriority()); assertTrue(Arrays.equals(evt.getBody(), e.getBody())); hdfs.close(); } /** * Test that pathnames are being correctly substituted. * * @throws InterruptedException */ @Test public void testTaggedWrites() throws IOException, InterruptedException { FlumeConfiguration conf = FlumeConfiguration.get(); String template = "file:///tmp/testfile-%{mytag}"; String real1 = "file:///tmp/testfile-one"; String real2 = "file:///tmp/testfile-two"; DFSEventSink sink = new DFSEventSink(template); Event e1 = new EventImpl("EVENT1".getBytes()); e1.set("mytag", "one".getBytes()); Event e2 = new EventImpl("EVENT2".getBytes()); e2.set("mytag", "two".getBytes()); sink.open(); sink.append(e1); sink.append(e2); sink.close(); Path path1 = new Path(real1); Path path2 = new Path(real2); FileSystem hdfs = path1.getFileSystem(conf); FileStatus stats = hdfs.getFileStatus(path1); assertTrue("File " + real1 + " not found but expected", stats.getLen() > 0); stats = hdfs.getFileStatus(new Path(real2)); assertTrue("File " + real2 + " not found but expected", stats.getLen() > 0); SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path1, conf); WriteableEventKey k = new WriteableEventKey(); WriteableEvent evt = new WriteableEvent(); r.next(k, evt); assertEquals(evt.getTimestamp(), e1.getTimestamp()); assertEquals(evt.getNanos(), e1.getNanos()); assertEquals(evt.getPriority(), e1.getPriority()); assertTrue(Arrays.equals(evt.getBody(), e1.getBody())); r = new SequenceFile.Reader(hdfs, path2, conf); k = new WriteableEventKey(); evt = new WriteableEvent(); r.next(k, evt); assertEquals(evt.getTimestamp(), e2.getTimestamp()); assertEquals(evt.getNanos(), e2.getNanos()); assertEquals(evt.getPriority(), e2.getPriority()); assertTrue(Arrays.equals(evt.getBody(), e2.getBody())); hdfs.close(); } // // Why does this test fail? It doesn't make sense. // // an old version used code similar to this and failed. Want to // // preserve this to see if it is a bug. // // TODO (jon) Why does this fail to write data? @Test public void testWhyFail() throws IOException { // There a was a failure case using : FlumeConfiguration conf = FlumeConfiguration.get(); Path path = new Path("file:///tmp/testfile"); FileSystem hdfs = path.getFileSystem(conf); // writing FSDataOutputStream dos = hdfs.create(path); hdfs.deleteOnExit(path); // this version's Writer has ownOutputStream=false. Writer writer = SequenceFile.createWriter(conf, dos, WriteableEventKey.class, WriteableEvent.class, SequenceFile.CompressionType.NONE, new DefaultCodec()); Event e = new EventImpl("EVENT".getBytes()); writer.append(new WriteableEventKey(e), new WriteableEvent(e)); writer.sync(); writer.close(); dos.close(); // It is strange that I have to close the underlying // FSDataOutputStream. // WTF: nothing written by this writer! FileStatus stats = hdfs.getFileStatus(path); assertTrue(stats.getLen() > 0); // it should have written something but it failed. } /** * No exception expected */ @Test public void testBadArgsParseOK() { SinkBuilder sb = DFSEventSink.builder(); sb.build(new Context(), "/foo/msgs"); } /** * wrong # args, exception expected */ @Test(expected = IllegalArgumentException.class) public void testBadArgsParseOkInstanceFail() { SinkBuilder sb = DFSEventSink.builder(); sb.build(new Context()); } /** * Failure occurs when opened due to permissions. * * @throws InterruptedException */ @Test public void testBadArgsOpenFail() throws IOException, InterruptedException { assumeTrue(!OSUtils.isWindowsOS()); SinkBuilder sb = DFSEventSink.builder(); EventSink snk = sb.build(new Context(), "/foo/msgs"); try { snk.open(); } catch (IOException e) { return; } fail("expected exception due to perms"); } }