Java tutorial
/* * Copyright (C) 2015 hops.io. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.hops.erasure_coding; import io.hops.metadata.hdfs.entity.EncodingPolicy; import io.hops.metadata.hdfs.entity.EncodingStatus; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ErasureCodingFileSystem; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockMissingException; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNodeUtil; import org.junit.Test; import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Random; import java.util.Set; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; public class TestErasureCodingManager extends ClusterTest { public static final Log LOG = LogFactory.getLog(TestErasureCodingManager.class); private static final int NUMBER_OF_DATANODES = 20; private static final int TEST_STRIPE_LENGTH = 10; private static final int TEST_PARITY_LENGTH = 6; private static final int TEST_STRIPE_COUNT = 2; private static final int TEST_BLOCK_COUNT = TEST_STRIPE_LENGTH * TEST_STRIPE_COUNT; private HdfsConfiguration conf; private final long seed = 0xDEADBEEFL; private final Path testFile = new Path("/test_file"); private DistributedFileSystem dfs; public TestErasureCodingManager() { conf = new HdfsConfiguration(); conf.setLong(DFS_BLOCK_SIZE_KEY, DFS_TEST_BLOCK_SIZE); conf.setInt(DFS_REPLICATION_KEY, 1); conf.set(DFSConfigKeys.ERASURE_CODING_CODECS_KEY, Util.JSON_CODEC_ARRAY); conf.setBoolean(DFSConfigKeys.ERASURE_CODING_ENABLED_KEY, true); conf.set(DFSConfigKeys.ENCODING_MANAGER_CLASSNAME_KEY, DFSConfigKeys.DEFAULT_ENCODING_MANAGER_CLASSNAME); conf.set(DFSConfigKeys.BLOCK_REPAIR_MANAGER_CLASSNAME_KEY, DFSConfigKeys.DEFAULT_BLOCK_REPAIR_MANAGER_CLASSNAME); conf.setInt(DFSConfigKeys.RECHECK_INTERVAL_KEY, 20 * 1000); conf.setInt("dfs.blockreport.intervalMsec", 30 * 1000); conf.setInt(DFSConfigKeys.REPAIR_DELAY_KEY, 10 * 1000); conf.setInt(DFSConfigKeys.PARITY_REPAIR_DELAY_KEY, 10 * 1000); conf.setClass("fs.hdfs.impl", ErasureCodingFileSystem.class, FileSystem.class); // Make sure it works with ecfs } @Override protected Configuration getConfig() { return conf; } @Override public void setUp() throws Exception { cluster = new MiniDFSCluster.Builder(getConfig()).numDataNodes(NUMBER_OF_DATANODES).build(); cluster.waitActive(); fs = FileSystem.get(conf); dfs = (DistributedFileSystem) ((ErasureCodingFileSystem) fs).getFileSystem(); FileStatus[] files = fs.globStatus(new Path("/*")); for (FileStatus file : files) { fs.delete(file.getPath(), true); } } @Test public void testEncoding() throws IOException, InterruptedException { Codec.initializeCodecs(getConfig()); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy); EncodingStatus status; while (!(status = dfs.getEncodingStatus(testFile.toUri().getPath())).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } assertEquals(status.getParityStatus(), EncodingStatus.ParityStatus.HEALTHY); Path parityPath = new Path(conf.get(DFSConfigKeys.PARITY_FOLDER, DFSConfigKeys.DEFAULT_PARITY_FOLDER), status.getParityFileName()); assertTrue(dfs.exists(parityPath)); assertFalse(status.getRevoked()); assertEquals(policy, status.getEncodingPolicy()); } @Test public void testLateEncoding() throws IOException { Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); dfs.encodeFile(testFile.toUri().getPath(), policy); EncodingStatus encodingStatus; while (!(encodingStatus = dfs.getEncodingStatus(testFile.toUri().getPath())).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } FileStatus fileStatus = dfs.getFileStatus(testFile); Path parityFile = new Path(conf.get(DFSConfigKeys.PARITY_FOLDER, DFSConfigKeys.DEFAULT_PARITY_FOLDER), encodingStatus.getParityFileName()); FileStatus parityStatus = dfs.getFileStatus(parityFile); BlockLocation[] blockLocations = dfs.getFileBlockLocations(fileStatus, 0, TEST_STRIPE_LENGTH * DFS_TEST_BLOCK_SIZE); BlockLocation[] parityBlockLocations = dfs.getFileBlockLocations(parityStatus, 0, TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE); Set<String> set = new HashSet<String>(); for (BlockLocation blockLocation : blockLocations) { String host = blockLocation.getNames()[0]; if (set.contains(host)) { fail("Duplicated location " + Arrays.toString(blockLocation.getNames())); } set.add(host); } for (BlockLocation blockLocation : parityBlockLocations) { String host = blockLocation.getNames()[0]; if (set.contains(host)) { fail("Duplicated location " + Arrays.toString(blockLocation.getNames())); } set.add(host); } } @Test public void testRevoke() throws IOException, InterruptedException { Codec.initializeCodecs(getConfig()); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy); EncodingStatus status; while (!(status = dfs.getEncodingStatus(testFile.toUri().getPath())).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } dfs.revokeEncoding(testFile.toUri().getPath(), (short) 2); while (dfs.getEncodingStatus(testFile.toUri().getPath()).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } Path parityPath = new Path(conf.get(DFSConfigKeys.PARITY_FOLDER, DFSConfigKeys.DEFAULT_PARITY_FOLDER), status.getParityFileName()); assertFalse(dfs.exists(parityPath)); assertEquals(2, dfs.getFileStatus(testFile).getReplication()); } @Test public void testDelete() throws IOException, InterruptedException { Codec.initializeCodecs(getConfig()); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy); EncodingStatus status; while (!(status = dfs.getEncodingStatus(testFile.toUri().getPath())).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } dfs.delete(testFile, false); Thread.sleep(2 * conf.getInt(DFSConfigKeys.RECHECK_INTERVAL_KEY, DFSConfigKeys.DEFAULT_RECHECK_INTERVAL)); Path parityPath = new Path(conf.get(DFSConfigKeys.PARITY_FOLDER, DFSConfigKeys.DEFAULT_PARITY_FOLDER), status.getParityFileName()); assertFalse(dfs.exists(parityPath)); } @Test public void testSourceRepair() throws IOException, InterruptedException { Codec.initializeCodecs(getConfig()); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy); FileStatus testFileStatus = dfs.getFileStatus(testFile); while (!dfs.getEncodingStatus(testFile.toUri().getPath()).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } Thread.sleep(2 * conf.getInt("dfs.blockreport.intervalMsec", 0)); EncodingStatus status = dfs.getEncodingStatus(testFile.toUri().getPath()); Path parityPath = new Path("/parity/" + status.getParityFileName()); FileStatus parityStatus = dfs.getFileStatus(parityPath); assertEquals(parityStatus.getLen(), TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE); try { FSDataInputStream in = dfs.open(parityPath); byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE]; in.readFully(0, buff); } catch (BlockMissingException e) { LOG.error("Reading parity failed", e); fail("Parity could not be read."); } String path = testFileStatus.getPath().toUri().getPath(); int blockToLoose = new Random(seed) .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize())); LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose); DataNodeUtil.loseBlock(getCluster(), lb); LOG.info("Losing block " + lb.toString()); Thread.sleep(2 * conf.getInt("dfs.blockreport.intervalMsec", 0) + 2 * conf.getInt(DFSConfigKeys.RECHECK_INTERVAL_KEY, 0)); while (true) { Thread.sleep(10000); EncodingStatus status2 = dfs.getEncodingStatus(testFile.toUri().getPath()); LOG.info("Current status is " + status2.getStatus()); if (status2.getStatus() == EncodingStatus.Status.ENCODED) { break; } } try { FSDataInputStream in = dfs.open(testFile); byte[] buff = new byte[TEST_BLOCK_COUNT * DFS_TEST_BLOCK_SIZE]; in.readFully(0, buff); } catch (BlockMissingException e) { fail("Repair failed. Missing a block."); } } @Test public void testParityRepair() throws IOException, InterruptedException { Codec.initializeCodecs(getConfig()); EncodingPolicy policy = new EncodingPolicy("src", (short) 1); Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE, policy); while (!dfs.getEncodingStatus(testFile.toUri().getPath()).isEncoded()) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Wait for encoding thread was interrupted."); } } Thread.sleep(2 * conf.getInt("dfs.blockreport.intervalMsec", 0)); EncodingStatus status = dfs.getEncodingStatus(testFile.toUri().getPath()); Path parityPath = new Path(conf.get(DFSConfigKeys.PARITY_FOLDER, DFSConfigKeys.DEFAULT_PARITY_FOLDER), status.getParityFileName()); FileStatus parityStatus = dfs.getFileStatus(parityPath); assertEquals(parityStatus.getLen(), TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE); try { FSDataInputStream in = dfs.open(parityPath); byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE]; in.readFully(0, buff); } catch (BlockMissingException e) { LOG.error("Reading parity failed", e); fail("Parity could not be read."); } int blockToLoose = new Random(seed).nextInt((int) (parityStatus.getLen() / parityStatus.getBlockSize())); LocatedBlock lb = dfs.getClient().getLocatedBlocks(parityPath.toUri().getPath(), 0, Long.MAX_VALUE) .get(blockToLoose); DataNodeUtil.loseBlock(getCluster(), lb); LOG.info("Losing block " + lb.toString()); try { FSDataInputStream in = dfs.open(parityPath); byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE]; in.readFully(0, buff); fail("Successfully read parity file which should have been broken."); } catch (BlockMissingException e) { } Thread.sleep(2 * conf.getInt("dfs.blockreport.intervalMsec", 0) + 2 * conf.getInt(DFSConfigKeys.RECHECK_INTERVAL_KEY, 0)); while (true) { Thread.sleep(10000); EncodingStatus status2 = dfs.getEncodingStatus(testFile.toUri().getPath()); LOG.info("Current status is " + status2); if (status2.getParityStatus() == EncodingStatus.ParityStatus.HEALTHY) { break; } } try { FSDataInputStream in = dfs.open(parityPath); byte[] buff = new byte[TEST_STRIPE_COUNT * TEST_PARITY_LENGTH * DFS_TEST_BLOCK_SIZE]; in.readFully(0, buff); } catch (BlockMissingException e) { fail("Repair failed. Missing a block."); } } @Override public void tearDown() throws Exception { fs.close(); cluster.shutdown(); } }