Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.raid; import java.io.File; import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.zip.CRC32; import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedRaidFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.RaidDFSUtil; import org.apache.hadoop.hdfs.TestRaidDfs; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.raid.DirectoryStripeReader.BlockInfo; public class TestDirectoryRaidDfs extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", "build/contrib/raid/test/data")) .getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestDirectoryRaidDecoder"); final static int NUM_DATANODES = 3; static { ParityFilePair.disableCacheUsedInTestOnly(); } final static long blockSize = 8192L; final static long halfBlock = blockSize / 2; final static long[][] fileSizes = { //Small file directory { 2000L, 3000L, 2000L }, { 3000L, 3000L, 3000L }, { 511L, 3584L, 3000L, 1234L, 512L, 1234L, 3000L }, //Identical Block Size File Directory { 1000L, blockSize, 2 * blockSize, 4000L }, { blockSize, blockSize, 4 * blockSize + 1 }, { blockSize + halfBlock, 2 * blockSize + halfBlock, halfBlock }, { blockSize + 1, 7 * blockSize + 1, 2 * blockSize + 1, 3 * blockSize + 1 }, //Different Block Size File Directory { 1000, blockSize, 2 * blockSize, 2 * blockSize + 1 }, { blockSize, 2 * blockSize, 3 * blockSize, 4 * blockSize }, { blockSize + 1, 9 * blockSize + 1, 2 * blockSize + 1, blockSize + 1 } }; final static long[][] blockSizes = { // Small file directory { blockSize, blockSize, blockSize }, { blockSize, blockSize, blockSize }, { blockSize, blockSize, blockSize, blockSize, blockSize, blockSize, blockSize }, //Identical Block Size File Directory { blockSize, blockSize, blockSize, blockSize }, { blockSize, blockSize, blockSize }, { blockSize, blockSize, blockSize }, { blockSize, blockSize, blockSize, blockSize }, //Different Block Size File Directory { blockSize, blockSize, 2 * blockSize, blockSize }, { blockSize, 2 * blockSize, 3 * blockSize, blockSize }, { blockSize, 2 * blockSize, 3 * blockSize, blockSize } }; Configuration conf; String namenode = null; String hftp = null; MiniDFSCluster dfs = null; FileSystem fileSys = null; String jobTrackerName = null; Codec codec; int stripeLength; public static void setupStripeStore(Configuration conf, FileSystem fs) { final String STRIPE_STORE_DIR = new File(TEST_DIR, "stripe_store." + System.currentTimeMillis()) .getAbsolutePath(); conf.set(RaidNode.RAID_STRIPE_STORE_CLASS_KEY, "org.apache.hadoop.raid.LocalStripeStore"); fs.getConf().set(RaidNode.RAID_STRIPE_STORE_CLASS_KEY, "org.apache.hadoop.raid.LocalStripeStore"); conf.set(LocalStripeStore.LOCAL_STRIPE_STORE_DIR_KEY + "." + fs.getUri().getAuthority(), STRIPE_STORE_DIR); fs.getConf().set(LocalStripeStore.LOCAL_STRIPE_STORE_DIR_KEY + "." + fs.getUri().getAuthority(), STRIPE_STORE_DIR); RaidNode.createStripeStore(conf, true, null); } private void mySetup(String erasureCode, int rsParityLength) throws Exception { new File(TEST_DIR).mkdirs(); // Make sure data directory exists conf = new Configuration(); conf.setInt("raid.encoder.bufsize", 128); conf.setInt("raid.decoder.bufsize", 128); Utils.loadTestCodecs(conf, stripeLength, stripeLength, 1, rsParityLength, "/destraid", "/destraidrs", false, true); codec = Codec.getCodec(erasureCode); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // Reduce run time for the test. conf.setInt("dfs.client.max.block.acquire.failures", 1); conf.setInt("dfs.client.baseTimeWindow.waitOn.BlockMissingException", 10); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); // Avoid datanode putting blocks under subdir directory. Corruptblock function // can only corrupt blocks under the current directory conf.setInt("dfs.datanode.numblocks", 1000); dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); setupStripeStore(conf, fileSys); namenode = fileSys.getUri().toString(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); //Don't allow empty file to be raid conf.setLong(RaidNode.MINIMUM_RAIDABLE_FILESIZE_KEY, 1L); } private void myTearDown() throws Exception { if (dfs != null) { dfs.shutdown(); } } static private DistributedRaidFileSystem getRaidFS(FileSystem fileSys, Configuration conf) throws IOException { DistributedFileSystem dfs = (DistributedFileSystem) fileSys; Configuration clientConf = new Configuration(conf); clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); clientConf.setBoolean("fs.hdfs.impl.disable.cache", true); URI dfsUri = dfs.getUri(); return (DistributedRaidFileSystem) FileSystem.get(dfsUri, clientConf); } static public void corruptBlocksInDirectory(Configuration conf, Path srcDir, long[] crcs, Integer[] listBlockNumToCorrupt, FileSystem fileSys, MiniDFSCluster cluster, boolean validate, boolean reportBadBlocks) throws IOException { long[] lengths = new long[crcs.length]; // Get all block Info; ArrayList<BlockInfo> blocks = new ArrayList<BlockInfo>(); List<FileStatus> lfs = RaidNode.listDirectoryRaidFileStatus(conf, fileSys, srcDir); assertNotNull(lfs); for (int fid = 0; fid < lfs.size(); fid++) { FileStatus fsStat = lfs.get(fid); long numBlock = RaidNode.getNumBlocks(fsStat); for (int bid = 0; bid < numBlock; bid++) { blocks.add(new BlockInfo(fid, bid)); } lengths[fid] = fsStat.getLen(); } HashSet<Integer> affectedFiles = new HashSet<Integer>(); HashSet<Integer> affectedBlocks = new HashSet<Integer>(); // corrupt blocks for (int blockNumToCorrupt : listBlockNumToCorrupt) { if (blockNumToCorrupt >= blocks.size()) { continue; } BlockInfo bi = null; int blockIndex = blockNumToCorrupt; if (blockNumToCorrupt < 0) { blockIndex = blocks.size() + blockNumToCorrupt; if (blockIndex < 0) { continue; } } if (affectedBlocks.contains(blockIndex)) { continue; } affectedBlocks.add(blockIndex); bi = blocks.get(blockIndex); FileStatus srcFileFs = lfs.get(bi.fileIdx); Path srcFile = srcFileFs.getPath(); LOG.info("Corrupt block " + bi.blockId + " of file " + srcFile); LocatedBlocks locations = RaidDFSUtil.getBlockLocations((DistributedFileSystem) fileSys, srcFile.toUri().getPath(), 0L, srcFileFs.getLen()); TestRaidDfs.corruptBlock(srcFile, locations.get(bi.blockId).getBlock(), NUM_DATANODES, true, cluster); if (reportBadBlocks) { cluster.getNameNode().reportBadBlocks(new LocatedBlock[] { locations.get(bi.blockId) }); } affectedFiles.add(bi.fileIdx); } // validate files if (validate) { DistributedRaidFileSystem raidfs = getRaidFS(fileSys, conf); for (Integer fid : affectedFiles) { FileStatus stat = lfs.get(fid); assertTrue(TestRaidDfs.validateFile(raidfs, stat.getPath(), lengths[fid], crcs[fid])); // test readFully byte[] filebytes = new byte[(int) stat.getLen()]; FSDataInputStream stm = raidfs.open(stat.getPath()); stm.readFully(0, filebytes); CRC32 crc = new CRC32(); crc.update(filebytes, 0, filebytes.length); assertEquals(crcs[fid], crc.getValue()); } } } /** * Create a bunch of files, corrupt blocks in some of them and ensure that * corrupted files can be read through DistributedRaidFileSystem */ public void testRaidDirDfs(String code, Integer[][] corrupt) throws Exception { LOG.info("Test testRaidDirDfs for " + code + " started."); stripeLength = 3; try { mySetup(code, 2); for (int i = 0; i < corrupt.length; i++) { Path srcDir = new Path("/user/dhruba/dir" + i); assert fileSizes.length == blockSizes.length; Codec curCodec = codec; for (int j = 0; j < fileSizes.length; j++) { LOG.info(" Test " + code + " at corrupt scenario " + i + " files scenario " + j); assert fileSizes[j].length == blockSizes[j].length; long[] crcs = new long[fileSizes[j].length]; int[] seeds = new int[fileSizes[j].length]; Path parityDir = new Path(codec.parityDirectory); RaidDFSUtil.cleanUp(fileSys, srcDir); RaidDFSUtil.cleanUp(fileSys, parityDir); TestRaidDfs.createTestFiles(srcDir, fileSizes[j], blockSizes[j], crcs, seeds, fileSys, (short) 1); assertTrue( RaidNode.doRaid(conf, fileSys.getFileStatus(srcDir), new Path(curCodec.parityDirectory), curCodec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1)); corruptBlocksInDirectory(conf, srcDir, crcs, corrupt[i], fileSys, dfs, true, false); RaidDFSUtil.cleanUp(fileSys, srcDir); RaidDFSUtil.cleanUp(fileSys, new Path(curCodec.parityDirectory)); RaidDFSUtil.cleanUp(fileSys, new Path("/tmp")); } } } catch (Exception e) { LOG.info("testRaidDirDfs Exception " + e.getMessage(), e); throw e; } finally { myTearDown(); } LOG.info("Test testRaidDirDfs for " + code + " completed."); } public void testXORRaidDirDfs() throws Exception { testRaidDirDfs("xor", new Integer[][] { { 0 }, { 1 }, { 2 }, { -3 }, { -2 }, { -1 } }); } public void testTooManyErrorsDecode() throws Exception { LOG.info("testTooManyErrorsDecode start"); long blockSize = 8192L; stripeLength = 3; mySetup("xor", 1); long[][] fsizes = { { 2000L, 3000L, 2000L }, { blockSize + 1, blockSize + 1 }, { 2 * blockSize, blockSize + blockSize / 2 } }; long[][] bsizes = { { blockSize, blockSize, blockSize }, { blockSize, blockSize }, { 2 * blockSize, blockSize } }; Integer[][] corrupts = { { 0, 1 }, { 0, 2 }, { 1, 2 } }; try { for (String code : RaidDFSUtil.codes) { Codec curCodec = Codec.getCodec(code); Path srcDir = new Path("/user/dhruba/" + code); for (int i = 0; i < corrupts.length; i++) { for (int j = 0; j < fsizes.length; j++) { long[] crcs = new long[fsizes[j].length]; int[] seeds = new int[fsizes[j].length]; Path parityDir = new Path(codec.parityDirectory); RaidDFSUtil.cleanUp(fileSys, srcDir); RaidDFSUtil.cleanUp(fileSys, parityDir); TestRaidDfs.createTestFiles(srcDir, fsizes[j], bsizes[j], crcs, seeds, fileSys, (short) 1); assertTrue(RaidNode.doRaid(conf, fileSys.getFileStatus(srcDir), new Path(curCodec.parityDirectory), curCodec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1)); boolean expectedExceptionThrown = false; try { corruptBlocksInDirectory(conf, srcDir, crcs, corrupts[i], fileSys, dfs, true, false); // Should not reach. } catch (IOException e) { LOG.info("Expected exception caught" + e); expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } } } LOG.info("testTooManyErrorsDecode complete"); } finally { myTearDown(); } } public void testTooManyErrorsEncode() throws Exception { LOG.info("testTooManyErrorsEncode complete"); stripeLength = 3; mySetup("xor", 1); // Encoding should fail when even one block is corrupt. Random rand = new Random(); try { for (String code : RaidDFSUtil.codes) { Codec curCodec = Codec.getCodec(code); Path srcDir = new Path("/user/dhruba/" + code); for (int j = 0; j < fileSizes.length; j++) { long[] crcs = new long[fileSizes[j].length]; int[] seeds = new int[fileSizes[j].length]; Path parityDir = new Path(codec.parityDirectory); RaidDFSUtil.cleanUp(fileSys, srcDir); RaidDFSUtil.cleanUp(fileSys, parityDir); TestRaidDfs.createTestFiles(srcDir, fileSizes[j], blockSizes[j], crcs, seeds, fileSys, (short) 1); corruptBlocksInDirectory(conf, srcDir, crcs, new Integer[] { rand.nextInt() % 3 }, fileSys, dfs, false, false); boolean expectedExceptionThrown = false; try { RaidNode.doRaid(conf, fileSys.getFileStatus(srcDir), new Path(curCodec.parityDirectory), curCodec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1); // Should not reach. } catch (IOException e) { LOG.info("Expected exception caught" + e); expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } } LOG.info("testTooManyErrorsEncode complete"); } finally { myTearDown(); } } }