Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.inmobi.conduit.distcp.tools; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.Credentials; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import com.inmobi.conduit.distcp.tools.CopyListing; import com.inmobi.conduit.distcp.tools.DistCpOptions; import com.inmobi.conduit.distcp.tools.FileBasedCopyListing; import com.inmobi.conduit.distcp.tools.util.TestDistCpUtils; import java.io.IOException; import java.io.OutputStream; import java.util.HashMap; import java.util.Map; public class TestFileBasedCopyListing { private static final Log LOG = LogFactory.getLog(TestFileBasedCopyListing.class); private static final Credentials CREDENTIALS = new Credentials(); private static final Configuration config = new Configuration(); private static MiniDFSCluster cluster; private static FileSystem fs; @BeforeClass public static void create() throws IOException { cluster = new MiniDFSCluster(config, 1, true, null); fs = cluster.getFileSystem(); buildExpectedValuesMap(); } @AfterClass public static void destroy() { if (cluster != null) { cluster.shutdown(); } } private static Map<String, String> map = new HashMap<String, String>(); private static void buildExpectedValuesMap() { map.put("/file1", "/tmp/singlefile1/file1"); map.put("/file2", "/tmp/singlefile2/file2"); map.put("/file3", "/tmp/multifile/file3"); map.put("/file4", "/tmp/multifile/file4"); map.put("/file5", "/tmp/multifile/file5"); map.put("/multifile/file3", "/tmp/multifile/file3"); map.put("/multifile/file4", "/tmp/multifile/file4"); map.put("/multifile/file5", "/tmp/multifile/file5"); map.put("/Ufile3", "/tmp/Umultifile/Ufile3"); map.put("/Ufile4", "/tmp/Umultifile/Ufile4"); map.put("/Ufile5", "/tmp/Umultifile/Ufile5"); map.put("/dir1", "/tmp/singledir/dir1"); map.put("/singledir/dir1", "/tmp/singledir/dir1"); map.put("/dir2", "/tmp/singledir/dir2"); map.put("/singledir/dir2", "/tmp/singledir/dir2"); map.put("/Udir1", "/tmp/Usingledir/Udir1"); map.put("/Udir2", "/tmp/Usingledir/Udir2"); map.put("/dir2/file6", "/tmp/singledir/dir2/file6"); map.put("/singledir/dir2/file6", "/tmp/singledir/dir2/file6"); map.put("/file7", "/tmp/singledir1/dir3/file7"); map.put("/file8", "/tmp/singledir1/dir3/file8"); map.put("/file9", "/tmp/singledir1/dir3/file9"); map.put("/dir3/file7", "/tmp/singledir1/dir3/file7"); map.put("/dir3/file8", "/tmp/singledir1/dir3/file8"); map.put("/dir3/file9", "/tmp/singledir1/dir3/file9"); map.put("/Ufile7", "/tmp/Usingledir1/Udir3/Ufile7"); map.put("/Ufile8", "/tmp/Usingledir1/Udir3/Ufile8"); map.put("/Ufile9", "/tmp/Usingledir1/Udir3/Ufile9"); } @Test public void testSingleFileMissingTarget() { caseSingleFileMissingTarget(false); caseSingleFileMissingTarget(true); } private void caseSingleFileMissingTarget(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/singlefile1/file1"); createFiles("/tmp/singlefile1/file1"); runTest(listFile, target, sync); checkResult(listFile, 0); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testSingleFileTargetFile() { caseSingleFileTargetFile(false); caseSingleFileTargetFile(true); } private void caseSingleFileTargetFile(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/singlefile1/file1"); createFiles("/tmp/singlefile1/file1", target.toString()); runTest(listFile, target, sync); checkResult(listFile, 0); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testSingleFileTargetDir() { caseSingleFileTargetDir(false); caseSingleFileTargetDir(true); } private void caseSingleFileTargetDir(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/singlefile2/file2"); createFiles("/tmp/singlefile2/file2"); mkdirs(target.toString()); runTest(listFile, target, sync); checkResult(listFile, 1); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testSingleDirTargetMissing() { caseSingleDirTargetMissing(false); caseSingleDirTargetMissing(true); } private void caseSingleDirTargetMissing(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/singledir"); mkdirs("/tmp/singledir/dir1"); runTest(listFile, target, sync); checkResult(listFile, 1); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testSingleDirTargetPresent() { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/singledir"); mkdirs("/tmp/singledir/dir1"); mkdirs(target.toString()); runTest(listFile, target); checkResult(listFile, 1); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testUpdateSingleDirTargetPresent() { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/Usingledir"); mkdirs("/tmp/Usingledir/Udir1"); mkdirs(target.toString()); runTest(listFile, target, true); checkResult(listFile, 1); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testMultiFileTargetPresent() { caseMultiFileTargetPresent(false); caseMultiFileTargetPresent(true); } private void caseMultiFileTargetPresent(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); mkdirs(target.toString()); runTest(listFile, target, sync); checkResult(listFile, 3); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testMultiFileTargetMissing() { caseMultiFileTargetMissing(false); caseMultiFileTargetMissing(true); } private void caseMultiFileTargetMissing(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); runTest(listFile, target, sync); checkResult(listFile, 3); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testMultiDirTargetPresent() { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/multifile", "/tmp/singledir"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); mkdirs(target.toString(), "/tmp/singledir/dir1"); runTest(listFile, target); checkResult(listFile, 4); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testUpdateMultiDirTargetPresent() { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/Umultifile", "/tmp/Usingledir"); createFiles("/tmp/Umultifile/Ufile3", "/tmp/Umultifile/Ufile4", "/tmp/Umultifile/Ufile5"); mkdirs(target.toString(), "/tmp/Usingledir/Udir1"); runTest(listFile, target, true); checkResult(listFile, 4); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testMultiDirTargetMissing() { caseMultiDirTargetMissing(false); caseMultiDirTargetMissing(true); } private void caseMultiDirTargetMissing(boolean sync) { try { Path listFile = new Path("/tmp/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/multifile", "/tmp/singledir"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); mkdirs("/tmp/singledir/dir1"); runTest(listFile, target, sync); checkResult(listFile, 4); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } } @Test public void testGlobTargetMissingSingleLevel() { caseGlobTargetMissingSingleLevel(false); caseGlobTargetMissingSingleLevel(true); } private void caseGlobTargetMissingSingleLevel(boolean sync) { try { Path listFile = new Path("/tmp1/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/*"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); createFiles("/tmp/singledir/dir2/file6"); runTest(listFile, target, sync); checkResult(listFile, 5); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); TestDistCpUtils.delete(fs, "/tmp1"); } } @Test public void testGlobTargetMissingMultiLevel() { caseGlobTargetMissingMultiLevel(false); caseGlobTargetMissingMultiLevel(true); } private void caseGlobTargetMissingMultiLevel(boolean sync) { try { Path listFile = new Path("/tmp1/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/*/*"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); createFiles("/tmp/singledir1/dir3/file7", "/tmp/singledir1/dir3/file8", "/tmp/singledir1/dir3/file9"); runTest(listFile, target, sync); checkResult(listFile, 6); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); TestDistCpUtils.delete(fs, "/tmp1"); } } @Test public void testGlobTargetDirMultiLevel() { try { Path listFile = new Path("/tmp1/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/*/*"); createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5"); createFiles("/tmp/singledir1/dir3/file7", "/tmp/singledir1/dir3/file8", "/tmp/singledir1/dir3/file9"); mkdirs(target.toString()); runTest(listFile, target); checkResult(listFile, 6); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); TestDistCpUtils.delete(fs, "/tmp1"); } } @Test public void testUpdateGlobTargetDirMultiLevel() { try { Path listFile = new Path("/tmp1/listing"); Path target = new Path("/tmp/target"); addEntries(listFile, "/tmp/*/*"); createFiles("/tmp/Umultifile/Ufile3", "/tmp/Umultifile/Ufile4", "/tmp/Umultifile/Ufile5"); createFiles("/tmp/Usingledir1/Udir3/Ufile7", "/tmp/Usingledir1/Udir3/Ufile8", "/tmp/Usingledir1/Udir3/Ufile9"); mkdirs(target.toString()); runTest(listFile, target, true); checkResult(listFile, 6); } catch (IOException e) { LOG.error("Exception encountered while testing build listing", e); Assert.fail("build listing failure"); } finally { TestDistCpUtils.delete(fs, "/tmp"); TestDistCpUtils.delete(fs, "/tmp1"); } } private void addEntries(Path listFile, String... entries) throws IOException { OutputStream out = fs.create(listFile); try { for (String entry : entries) { out.write(entry.getBytes()); out.write("\n".getBytes()); } } finally { out.close(); } } private void createFiles(String... entries) throws IOException { for (String entry : entries) { OutputStream out = fs.create(new Path(entry)); try { out.write(entry.getBytes()); out.write("\n".getBytes()); } finally { out.close(); } } } private void mkdirs(String... entries) throws IOException { for (String entry : entries) { fs.mkdirs(new Path(entry)); } } private void runTest(Path listFile, Path target) throws IOException { runTest(listFile, target, true); } private void runTest(Path listFile, Path target, boolean sync) throws IOException { CopyListing listing = new FileBasedCopyListing(config, CREDENTIALS); DistCpOptions options = new DistCpOptions(listFile, target); options.setSyncFolder(sync); listing.buildListing(listFile, options); } private void checkResult(Path listFile, int count) throws IOException { if (count == 0) { return; } int recCount = 0; SequenceFile.Reader reader = new SequenceFile.Reader(fs, listFile, config); try { Text relPath = new Text(); FileStatus fileStatus = new FileStatus(); while (reader.next(relPath, fileStatus)) { Assert.assertEquals(fileStatus.getPath().toUri().getPath(), map.get(relPath.toString())); recCount++; } } finally { IOUtils.closeStream(reader); } Assert.assertEquals(recCount, count); } }