Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.accumulo.core.client.rfile; import java.io.File; import java.io.IOException; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.SortedMap; import java.util.TreeMap; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; import org.apache.accumulo.core.file.rfile.RFile.Reader; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.io.Text; import org.junit.Assert; import org.junit.Test; import com.google.common.collect.ImmutableMap; public class RFileTest { // method created to foil findbugs... it was complaining ret val not used when it did not matter private void foo(boolean b) { } private String createTmpTestFile() throws IOException { File dir = new File(System.getProperty("user.dir") + "/target/rfile-test"); foo(dir.mkdirs()); File testFile = File.createTempFile("test", ".rf", dir); foo(testFile.delete()); return testFile.getAbsolutePath(); } String rowStr(int r) { return String.format("%06x", r); } String colStr(int c) { return String.format("%04x", c); } private SortedMap<Key, Value> createTestData(int rows, int families, int qualifiers) { return createTestData(0, rows, 0, families, qualifiers); } private SortedMap<Key, Value> createTestData(int startRow, int rows, int startFamily, int families, int qualifiers) { TreeMap<Key, Value> testData = new TreeMap<>(); for (int r = 0; r < rows; r++) { String row = rowStr(r + startRow); for (int f = 0; f < families; f++) { String fam = colStr(f + startFamily); for (int q = 0; q < qualifiers; q++) { String qual = colStr(q); Key k = new Key(row, fam, qual); testData.put(k, new Value((k.hashCode() + "").getBytes())); } } } return testData; } private String createRFile(SortedMap<Key, Value> testData) throws Exception { String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile) .withFileSystem(FileSystem.getLocal(new Configuration())).build()) { writer.append(testData.entrySet()); // TODO ensure compressors are returned } return testFile; } @Test public void testIndependance() throws Exception { // test to ensure two iterators allocated from same RFile scanner are independent. LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Range range1 = Range.exact(rowStr(5)); scanner.setRange(range1); Iterator<Entry<Key, Value>> scnIter1 = scanner.iterator(); Iterator<Entry<Key, Value>> mapIter1 = testData.subMap(range1.getStartKey(), range1.getEndKey()).entrySet() .iterator(); Range range2 = new Range(rowStr(3), true, rowStr(4), true); scanner.setRange(range2); Iterator<Entry<Key, Value>> scnIter2 = scanner.iterator(); Iterator<Entry<Key, Value>> mapIter2 = testData.subMap(range2.getStartKey(), range2.getEndKey()).entrySet() .iterator(); while (scnIter1.hasNext() || scnIter2.hasNext()) { if (scnIter1.hasNext()) { Assert.assertTrue(mapIter1.hasNext()); Assert.assertEquals(scnIter1.next(), mapIter1.next()); } else { Assert.assertFalse(mapIter1.hasNext()); } if (scnIter2.hasNext()) { Assert.assertTrue(mapIter2.hasNext()); Assert.assertEquals(scnIter2.next(), mapIter2.next()); } else { Assert.assertFalse(mapIter2.hasNext()); } } Assert.assertFalse(mapIter1.hasNext()); Assert.assertFalse(mapIter2.hasNext()); scanner.close(); } SortedMap<Key, Value> toMap(Scanner scanner) { TreeMap<Key, Value> map = new TreeMap<>(); for (Entry<Key, Value> entry : scanner) { map.put(entry.getKey(), entry.getValue()); } return map; } @Test public void testMultipleSources() throws Exception { SortedMap<Key, Value> testData1 = createTestData(10, 10, 10); SortedMap<Key, Value> testData2 = createTestData(0, 10, 0, 10, 10); String testFile1 = createRFile(testData1); String testFile2 = createRFile(testData2); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); Scanner scanner = RFile.newScanner().from(testFile1, testFile2).withFileSystem(localFs).build(); TreeMap<Key, Value> expected = new TreeMap<>(testData1); expected.putAll(testData2); Assert.assertEquals(expected, toMap(scanner)); Range range = new Range(rowStr(3), true, rowStr(14), true); scanner.setRange(range); Assert.assertEquals(expected.subMap(range.getStartKey(), range.getEndKey()), toMap(scanner)); scanner.close(); } @Test public void testWriterTableProperties() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); Map<String, String> props = new HashMap<>(); props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K"); props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), "1K"); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withTableProperties(props) .build(); SortedMap<Key, Value> testData1 = createTestData(10, 10, 10); writer.append(testData1.entrySet()); writer.close(); Reader reader = getReader(localFs, testFile); FileSKVIterator iiter = reader.getIndex(); int count = 0; while (iiter.hasTop()) { count++; iiter.next(); } // if settings are used then should create multiple index entries Assert.assertTrue(count > 10); reader.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertEquals(testData1, toMap(scanner)); scanner.close(); } @Test public void testLocalityGroups() throws Exception { SortedMap<Key, Value> testData1 = createTestData(0, 10, 0, 2, 10); SortedMap<Key, Value> testData2 = createTestData(0, 10, 2, 1, 10); SortedMap<Key, Value> defaultData = createTestData(0, 10, 3, 7, 10); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); writer.startNewLocalityGroup("z", colStr(0), colStr(1)); writer.append(testData1.entrySet()); writer.startNewLocalityGroup("h", colStr(2)); writer.append(testData2.entrySet()); writer.startDefaultLocalityGroup(); writer.append(defaultData.entrySet()); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); scanner.fetchColumnFamily(new Text(colStr(0))); scanner.fetchColumnFamily(new Text(colStr(1))); Assert.assertEquals(testData1, toMap(scanner)); scanner.clearColumns(); scanner.fetchColumnFamily(new Text(colStr(2))); Assert.assertEquals(testData2, toMap(scanner)); scanner.clearColumns(); for (int i = 3; i < 10; i++) { scanner.fetchColumnFamily(new Text(colStr(i))); } Assert.assertEquals(defaultData, toMap(scanner)); scanner.clearColumns(); Assert.assertEquals(createTestData(10, 10, 10), toMap(scanner)); scanner.close(); Reader reader = getReader(localFs, testFile); Map<String, ArrayList<ByteSequence>> lGroups = reader.getLocalityGroupCF(); Assert.assertTrue(lGroups.containsKey("z")); Assert.assertTrue(lGroups.get("z").size() == 2); Assert.assertTrue(lGroups.get("z").contains(new ArrayByteSequence(colStr(0)))); Assert.assertTrue(lGroups.get("z").contains(new ArrayByteSequence(colStr(1)))); Assert.assertTrue(lGroups.containsKey("h")); Assert.assertEquals(Arrays.asList(new ArrayByteSequence(colStr(2))), lGroups.get("h")); reader.close(); } @Test public void testIterators() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); IteratorSetting is = new IteratorSetting(50, "regex", RegExFilter.class); RegExFilter.setRegexs(is, ".*00000[78].*", null, null, null, false); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); scanner.addScanIterator(is); Assert.assertEquals(createTestData(7, 2, 0, 10, 10), toMap(scanner)); scanner.close(); } @Test public void testAuths() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1", "A&B"); Key k2 = new Key("r1", "f1", "q2", "A"); Key k3 = new Key("r1", "f1", "q3"); Value v1 = new Value("p".getBytes()); Value v2 = new Value("c".getBytes()); Value v3 = new Value("t".getBytes()); writer.append(k1, v1); writer.append(k2, v2); writer.append(k3, v3); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("A")).build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("A"), scanner.getAuthorizations()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("A", "B")).build(); Assert.assertEquals(ImmutableMap.of(k1, v1, k2, v2, k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("A", "B"), scanner.getAuthorizations()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("B")).build(); Assert.assertEquals(ImmutableMap.of(k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("B"), scanner.getAuthorizations()); scanner.close(); } @Test public void testNoSystemIters() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1"); k1.setTimestamp(3); Key k2 = new Key("r1", "f1", "q1"); k2.setTimestamp(6); k2.setDeleted(true); Value v1 = new Value("p".getBytes()); Value v2 = new Value("".getBytes()); writer.append(k2, v2); writer.append(k1, v1); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertFalse(scanner.iterator().hasNext()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withoutSystemIterators().build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k1, v1), toMap(scanner)); scanner.setRange(new Range("r2")); Assert.assertFalse(scanner.iterator().hasNext()); scanner.close(); } @Test public void testBounds() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); // set a lower bound row Range bounds = new Range(rowStr(3), false, null, true); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(4, 6, 0, 10, 10), toMap(scanner)); scanner.close(); // set an upper bound row bounds = new Range(null, false, rowStr(7), true); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(8, 10, 10), toMap(scanner)); scanner.close(); // set row bounds bounds = new Range(rowStr(3), false, rowStr(7), true); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(4, 4, 0, 10, 10), toMap(scanner)); scanner.close(); // set a row family bound bounds = Range.exact(rowStr(3), colStr(5)); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(3, 1, 5, 1, 10), toMap(scanner)); scanner.close(); } @Test public void testScannerTableProperties() throws Exception { NewTableConfiguration ntc = new NewTableConfiguration(); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1"); k1.setTimestamp(3); Key k2 = new Key("r1", "f1", "q1"); k2.setTimestamp(6); Value v1 = new Value("p".getBytes()); Value v2 = new Value("q".getBytes()); writer.append(k2, v2); writer.append(k1, v1); writer.close(); // pass in table config that has versioning iterator configured Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withTableProperties(ntc.getProperties()).build(); Assert.assertEquals(ImmutableMap.of(k2, v2), toMap(scanner)); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k1, v1), toMap(scanner)); scanner.close(); } @Test public void testSampling() throws Exception { SortedMap<Key, Value> testData1 = createTestData(1000, 2, 1); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); SamplerConfiguration sc = new SamplerConfiguration(RowSampler.class) .setOptions(ImmutableMap.of("hasher", "murmur3_32", "modulus", "19")); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withSampler(sc).build(); writer.append(testData1.entrySet()); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); scanner.setSamplerConfiguration(sc); RowSampler rowSampler = new RowSampler(); rowSampler.init(sc); SortedMap<Key, Value> sampleData = new TreeMap<>(); for (Entry<Key, Value> e : testData1.entrySet()) { if (rowSampler.accept(e.getKey())) { sampleData.put(e.getKey(), e.getValue()); } } Assert.assertTrue(sampleData.size() < testData1.size()); Assert.assertEquals(sampleData, toMap(scanner)); scanner.clearSamplerConfiguration(); Assert.assertEquals(testData1, toMap(scanner)); } @Test public void testAppendScanner() throws Exception { SortedMap<Key, Value> testData = createTestData(10000, 1, 1); String testFile = createRFile(testData); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); String testFile2 = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile2).build(); writer.append(scanner); writer.close(); scanner.close(); scanner = RFile.newScanner().from(testFile2).withFileSystem(localFs).build(); Assert.assertEquals(testData, toMap(scanner)); scanner.close(); } @Test public void testCache() throws Exception { SortedMap<Key, Value> testData = createTestData(10000, 1, 1); String testFile = createRFile(testData); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withIndexCache(1000000) .withDataCache(10000000).build(); Random rand = new Random(5); for (int i = 0; i < 100; i++) { int r = rand.nextInt(10000); scanner.setRange(new Range(rowStr(r))); Iterator<Entry<Key, Value>> iter = scanner.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(rowStr(r), iter.next().getKey().getRow().toString()); Assert.assertFalse(iter.hasNext()); } scanner.close(); } @Test(expected = IllegalArgumentException.class) public void testOutOfOrder() throws Exception { // test that exception declared in API is thrown Key k1 = new Key("r1", "f1", "q1"); Value v1 = new Value("1".getBytes()); Key k2 = new Key("r2", "f1", "q1"); Value v2 = new Value("2".getBytes()); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.append(k2, v2); writer.append(k1, v1); } } @Test(expected = IllegalArgumentException.class) public void testOutOfOrderIterable() throws Exception { // test that exception declared in API is thrown Key k1 = new Key("r1", "f1", "q1"); Value v1 = new Value("1".getBytes()); Key k2 = new Key("r2", "f1", "q1"); Value v2 = new Value("2".getBytes()); ArrayList<Entry<Key, Value>> data = new ArrayList<>(); data.add(new AbstractMap.SimpleEntry<>(k2, v2)); data.add(new AbstractMap.SimpleEntry<>(k1, v1)); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.append(data); } } @Test(expected = IllegalArgumentException.class) public void testBadVis() throws Exception { // this test has two purposes ensure an exception is thrown and ensure the exception document in the javadoc is thrown LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.startDefaultLocalityGroup(); Key k1 = new Key("r1", "f1", "q1", "(A&(B"); writer.append(k1, new Value("".getBytes())); } } @Test(expected = IllegalArgumentException.class) public void testBadVisIterable() throws Exception { // test append(iterable) method LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.startDefaultLocalityGroup(); Key k1 = new Key("r1", "f1", "q1", "(A&(B"); Entry<Key, Value> entry = new AbstractMap.SimpleEntry<>(k1, new Value("".getBytes())); writer.append(Collections.singletonList(entry)); } } @Test(expected = IllegalStateException.class) public void testDoubleStart() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.startDefaultLocalityGroup(); writer.startDefaultLocalityGroup(); } } @Test(expected = IllegalStateException.class) public void testAppendStartDefault() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.append(new Key("r1", "f1", "q1"), new Value("1".getBytes())); writer.startDefaultLocalityGroup(); } } @Test(expected = IllegalStateException.class) public void testStartAfter() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { Key k1 = new Key("r1", "f1", "q1"); writer.append(k1, new Value("".getBytes())); writer.startNewLocalityGroup("lg1", "fam1"); } } @Test(expected = IllegalArgumentException.class) public void testIllegalColumn() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.startNewLocalityGroup("lg1", "fam1"); Key k1 = new Key("r1", "f1", "q1"); // should not be able to append the column family f1 writer.append(k1, new Value("".getBytes())); } } @Test(expected = IllegalArgumentException.class) public void testWrongGroup() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { writer.startNewLocalityGroup("lg1", "fam1"); Key k1 = new Key("r1", "fam1", "q1"); writer.append(k1, new Value("".getBytes())); writer.startDefaultLocalityGroup(); // should not be able to append the column family fam1 to default locality group Key k2 = new Key("r1", "fam1", "q2"); writer.append(k2, new Value("".getBytes())); } } private Reader getReader(LocalFileSystem localFs, String testFile) throws IOException { Reader reader = (Reader) FileOperations.getInstance().newReaderBuilder().forFile(testFile) .inFileSystem(localFs, localFs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); return reader; } }