Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloudera.sqoop.io; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.File; import java.io.FileInputStream; import java.util.zip.GZIPInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.GzipCodec; import com.cloudera.sqoop.testutil.ImportJobTestCase; import junit.framework.TestCase; /** * Test that the splittable buffered writer system works. */ public class TestSplittableBufferedWriter extends TestCase { public static final Log LOG = LogFactory.getLog(TestSplittableBufferedWriter.class.getName()); private String getWriteDir() { return new File(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest").toString(); } private Path getWritePath() { return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest"); } /** Create the directory where we'll write our test files to; and * make sure it has no files in it. */ private void ensureEmptyWriteDir() throws IOException { FileSystem fs = FileSystem.getLocal(getConf()); Path writeDir = getWritePath(); fs.mkdirs(writeDir); FileStatus[] stats = fs.listStatus(writeDir); for (FileStatus stat : stats) { if (stat.isDir()) { fail("setUp(): Write directory " + writeDir + " contains subdirectories"); } LOG.debug("setUp(): Removing " + stat.getPath()); if (!fs.delete(stat.getPath(), false)) { fail("setUp(): Could not delete residual file " + stat.getPath()); } } if (!fs.exists(writeDir)) { fail("setUp: Could not create " + writeDir); } } public void setUp() throws IOException { ensureEmptyWriteDir(); } private Configuration getConf() { Configuration conf = new Configuration(); conf.set("fs.default.name", "file:///"); return conf; } /** Verifies contents of an InputStream. Closes the InputStream on * its way out. Fails the test if the file doesn't match the expected set * of lines. */ private void verifyFileContents(InputStream is, String[] lines) throws IOException { BufferedReader r = new BufferedReader(new InputStreamReader(is)); try { for (String expectedLine : lines) { String actualLine = r.readLine(); assertNotNull(actualLine); assertEquals("Input line mismatch", expectedLine, actualLine); } assertNull("Stream had additional contents after expected line", r.readLine()); } finally { r.close(); try { is.close(); } catch (IOException ioe) { // ignore IOE; may be closed by reader. } } } private void verifyFileExists(Path p) throws IOException { FileSystem fs = FileSystem.getLocal(getConf()); assertTrue("File not found: " + p, fs.exists(p)); } private void verifyFileDoesNotExist(Path p) throws IOException { FileSystem fs = FileSystem.getLocal(getConf()); assertFalse("File found: " + p + " and we did not expect it", fs.exists(p)); } public void testNonSplittingTextFile() throws IOException { SplittingOutputStream os = new SplittingOutputStream(getConf(), getWritePath(), "nonsplit-", 0, null); try { SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); try { w.allowSplit(); w.write("This is a string!"); w.newLine(); w.write("This is another string!"); w.allowSplit(); } finally { w.close(); } } finally { try { os.close(); } catch (IOException ioe) { // Ignored; may be thrown because w is already closed. } } // Ensure we made exactly one file. Path writePath = new Path(getWritePath(), "nonsplit-00000"); Path badPath = new Path(getWritePath(), "nonsplit-00001"); verifyFileExists(writePath); verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. // Now ensure all the data got there. String[] expectedLines = { "This is a string!", "This is another string!", }; InputStream fis = new FileInputStream(new File(getWriteDir(), "nonsplit-00000")); try { verifyFileContents(fis, expectedLines); } finally { try { fis.close(); } catch (IOException ioe) { // Ignored; may be closed by verifyFileContents(). } } } public void testNonSplittingGzipFile() throws IOException { SplittingOutputStream os = new SplittingOutputStream(getConf(), getWritePath(), "nonsplit-", 0, new GzipCodec()); SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); try { w.allowSplit(); w.write("This is a string!"); w.newLine(); w.write("This is another string!"); w.allowSplit(); } finally { w.close(); } // Ensure we made exactly one file. Path writePath = new Path(getWritePath(), "nonsplit-00000.gz"); Path badPath = new Path(getWritePath(), "nonsplit-00001.gz"); verifyFileExists(writePath); verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. // Now ensure all the data got there. String[] expectedLines = { "This is a string!", "This is another string!", }; verifyFileContents(new GZIPInputStream(new FileInputStream(new File(getWriteDir(), "nonsplit-00000.gz"))), expectedLines); } public void testSplittingTextFile() throws IOException { SplittingOutputStream os = new SplittingOutputStream(getConf(), getWritePath(), "split-", 10, null); try { SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); try { w.allowSplit(); w.write("This is a string!"); w.newLine(); w.write("This is another string!"); } finally { w.close(); } } finally { try { os.close(); } catch (IOException ioe) { // Ignored; may be thrown because w is already closed. } } // Ensure we made exactly two files. Path writePath = new Path(getWritePath(), "split-00000"); Path writePath2 = new Path(getWritePath(), "split-00001"); Path badPath = new Path(getWritePath(), "split-00002"); verifyFileExists(writePath); verifyFileExists(writePath2); verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. // Now ensure all the data got there. String[] expectedLines0 = { "This is a string!", }; InputStream fis = new FileInputStream(new File(getWriteDir(), "split-00000")); try { verifyFileContents(fis, expectedLines0); } finally { try { fis.close(); } catch (IOException ioe) { // ignored; may be generated because fis closed in verifyFileContents. } } String[] expectedLines1 = { "This is another string!", }; fis = new FileInputStream(new File(getWriteDir(), "split-00001")); try { verifyFileContents(fis, expectedLines1); } finally { try { fis.close(); } catch (IOException ioe) { // Ignored; may be thrown because it's closed in verifyFileContents. } } } public void testSplittingGzipFile() throws IOException { SplittingOutputStream os = new SplittingOutputStream(getConf(), getWritePath(), "splitz-", 3, new GzipCodec()); SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); try { w.write("This is a string!"); w.newLine(); w.write("This is another string!"); } finally { w.close(); } // Ensure we made exactly two files. Path writePath = new Path(getWritePath(), "splitz-00000.gz"); Path writePath2 = new Path(getWritePath(), "splitz-00001.gz"); Path badPath = new Path(getWritePath(), "splitz-00002.gz"); verifyFileExists(writePath); verifyFileExists(writePath2); verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. // Now ensure all the data got there. String[] expectedLines0 = { "This is a string!", }; verifyFileContents(new GZIPInputStream(new FileInputStream(new File(getWriteDir(), "splitz-00000.gz"))), expectedLines0); String[] expectedLines1 = { "This is another string!", }; verifyFileContents(new GZIPInputStream(new FileInputStream(new File(getWriteDir(), "splitz-00001.gz"))), expectedLines1); } }