List of usage examples for org.apache.hadoop.fs FileSystem getLocal
public static LocalFileSystem getLocal(Configuration conf) throws IOException
From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java
License:Apache License
@Test public void testMultipleFilesAndCache() throws Exception { SortedMap<Key, Value> testData = createTestData(100, 10, 10); List<String> files = Arrays.asList(createTmpTestFile(), createTmpTestFile(), createTmpTestFile()); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); for (int i = 0; i < files.size(); i++) { try (RFileWriter writer = RFile.newWriter().to(files.get(i)).withFileSystem(localFs).build()) { for (Entry<Key, Value> entry : testData.entrySet()) { if (entry.getKey().hashCode() % files.size() == i) { writer.append(entry.getKey(), entry.getValue()); }/*from w w w .jav a 2s. c o m*/ } } } Scanner scanner = RFile.newScanner().from(files.toArray(new String[files.size()])).withFileSystem(localFs) .withIndexCache(1000000).withDataCache(10000000).build(); assertEquals(testData, toMap(scanner)); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testIndependance() throws Exception { // test to ensure two iterators allocated from same RFile scanner are independent. LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Range range1 = Range.exact(rowStr(5)); scanner.setRange(range1);/*from ww w .j a v a 2s .c o m*/ Iterator<Entry<Key, Value>> scnIter1 = scanner.iterator(); Iterator<Entry<Key, Value>> mapIter1 = testData.subMap(range1.getStartKey(), range1.getEndKey()).entrySet() .iterator(); Range range2 = new Range(rowStr(3), true, rowStr(4), true); scanner.setRange(range2); Iterator<Entry<Key, Value>> scnIter2 = scanner.iterator(); Iterator<Entry<Key, Value>> mapIter2 = testData.subMap(range2.getStartKey(), range2.getEndKey()).entrySet() .iterator(); while (scnIter1.hasNext() || scnIter2.hasNext()) { if (scnIter1.hasNext()) { Assert.assertTrue(mapIter1.hasNext()); Assert.assertEquals(scnIter1.next(), mapIter1.next()); } else { Assert.assertFalse(mapIter1.hasNext()); } if (scnIter2.hasNext()) { Assert.assertTrue(mapIter2.hasNext()); Assert.assertEquals(scnIter2.next(), mapIter2.next()); } else { Assert.assertFalse(mapIter2.hasNext()); } } Assert.assertFalse(mapIter1.hasNext()); Assert.assertFalse(mapIter2.hasNext()); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testMultipleSources() throws Exception { SortedMap<Key, Value> testData1 = createTestData(10, 10, 10); SortedMap<Key, Value> testData2 = createTestData(0, 10, 0, 10, 10); String testFile1 = createRFile(testData1); String testFile2 = createRFile(testData2); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); Scanner scanner = RFile.newScanner().from(testFile1, testFile2).withFileSystem(localFs).build(); TreeMap<Key, Value> expected = new TreeMap<>(testData1); expected.putAll(testData2);/* w w w . jav a2s. c om*/ Assert.assertEquals(expected, toMap(scanner)); Range range = new Range(rowStr(3), true, rowStr(14), true); scanner.setRange(range); Assert.assertEquals(expected.subMap(range.getStartKey(), range.getEndKey()), toMap(scanner)); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testWriterTableProperties() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); Map<String, String> props = new HashMap<>(); props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K"); props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), "1K"); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withTableProperties(props) .build();/*from w ww .j ava2 s. c o m*/ SortedMap<Key, Value> testData1 = createTestData(10, 10, 10); writer.append(testData1.entrySet()); writer.close(); Reader reader = getReader(localFs, testFile); FileSKVIterator iiter = reader.getIndex(); int count = 0; while (iiter.hasTop()) { count++; iiter.next(); } // if settings are used then should create multiple index entries Assert.assertTrue(count > 10); reader.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertEquals(testData1, toMap(scanner)); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testLocalityGroups() throws Exception { SortedMap<Key, Value> testData1 = createTestData(0, 10, 0, 2, 10); SortedMap<Key, Value> testData2 = createTestData(0, 10, 2, 1, 10); SortedMap<Key, Value> defaultData = createTestData(0, 10, 3, 7, 10); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); writer.startNewLocalityGroup("z", colStr(0), colStr(1)); writer.append(testData1.entrySet()); writer.startNewLocalityGroup("h", colStr(2)); writer.append(testData2.entrySet()); writer.startDefaultLocalityGroup();// w w w . j av a 2 s. co m writer.append(defaultData.entrySet()); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); scanner.fetchColumnFamily(new Text(colStr(0))); scanner.fetchColumnFamily(new Text(colStr(1))); Assert.assertEquals(testData1, toMap(scanner)); scanner.clearColumns(); scanner.fetchColumnFamily(new Text(colStr(2))); Assert.assertEquals(testData2, toMap(scanner)); scanner.clearColumns(); for (int i = 3; i < 10; i++) { scanner.fetchColumnFamily(new Text(colStr(i))); } Assert.assertEquals(defaultData, toMap(scanner)); scanner.clearColumns(); Assert.assertEquals(createTestData(10, 10, 10), toMap(scanner)); scanner.close(); Reader reader = getReader(localFs, testFile); Map<String, ArrayList<ByteSequence>> lGroups = reader.getLocalityGroupCF(); Assert.assertTrue(lGroups.containsKey("z")); Assert.assertTrue(lGroups.get("z").size() == 2); Assert.assertTrue(lGroups.get("z").contains(new ArrayByteSequence(colStr(0)))); Assert.assertTrue(lGroups.get("z").contains(new ArrayByteSequence(colStr(1)))); Assert.assertTrue(lGroups.containsKey("h")); Assert.assertEquals(Arrays.asList(new ArrayByteSequence(colStr(2))), lGroups.get("h")); reader.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testIterators() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); IteratorSetting is = new IteratorSetting(50, "regex", RegExFilter.class); RegExFilter.setRegexs(is, ".*00000[78].*", null, null, null, false); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); scanner.addScanIterator(is);/* w w w. j a va 2 s . c o m*/ Assert.assertEquals(createTestData(7, 2, 0, 10, 10), toMap(scanner)); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testAuths() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1", "A&B"); Key k2 = new Key("r1", "f1", "q2", "A"); Key k3 = new Key("r1", "f1", "q3"); Value v1 = new Value("p".getBytes()); Value v2 = new Value("c".getBytes()); Value v3 = new Value("t".getBytes()); writer.append(k1, v1);//w w w .j a v a 2s . c o m writer.append(k2, v2); writer.append(k3, v3); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("A")).build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("A"), scanner.getAuthorizations()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("A", "B")).build(); Assert.assertEquals(ImmutableMap.of(k1, v1, k2, v2, k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("A", "B"), scanner.getAuthorizations()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withAuthorizations(new Authorizations("B")).build(); Assert.assertEquals(ImmutableMap.of(k3, v3), toMap(scanner)); Assert.assertEquals(new Authorizations("B"), scanner.getAuthorizations()); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testNoSystemIters() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1"); k1.setTimestamp(3);//w w w .j av a 2 s .co m Key k2 = new Key("r1", "f1", "q1"); k2.setTimestamp(6); k2.setDeleted(true); Value v1 = new Value("p".getBytes()); Value v2 = new Value("".getBytes()); writer.append(k2, v2); writer.append(k1, v1); writer.close(); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertFalse(scanner.iterator().hasNext()); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withoutSystemIterators().build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k1, v1), toMap(scanner)); scanner.setRange(new Range("r2")); Assert.assertFalse(scanner.iterator().hasNext()); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testBounds() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); SortedMap<Key, Value> testData = createTestData(10, 10, 10); String testFile = createRFile(testData); // set a lower bound row Range bounds = new Range(rowStr(3), false, null, true); Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(4, 6, 0, 10, 10), toMap(scanner)); scanner.close();//from w w w .ja v a2 s. c o m // set an upper bound row bounds = new Range(null, false, rowStr(7), true); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(8, 10, 10), toMap(scanner)); scanner.close(); // set row bounds bounds = new Range(rowStr(3), false, rowStr(7), true); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(4, 4, 0, 10, 10), toMap(scanner)); scanner.close(); // set a row family bound bounds = Range.exact(rowStr(3), colStr(5)); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withBounds(bounds).build(); Assert.assertEquals(createTestData(3, 1, 5, 1, 10), toMap(scanner)); scanner.close(); }
From source file:org.apache.accumulo.core.client.rfile.RFileTest.java
License:Apache License
@Test public void testScannerTableProperties() throws Exception { NewTableConfiguration ntc = new NewTableConfiguration(); LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build(); Key k1 = new Key("r1", "f1", "q1"); k1.setTimestamp(3);/*from w ww . j av a 2 s. c om*/ Key k2 = new Key("r1", "f1", "q1"); k2.setTimestamp(6); Value v1 = new Value("p".getBytes()); Value v2 = new Value("q".getBytes()); writer.append(k2, v2); writer.append(k1, v1); writer.close(); // pass in table config that has versioning iterator configured Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs) .withTableProperties(ntc.getProperties()).build(); Assert.assertEquals(ImmutableMap.of(k2, v2), toMap(scanner)); scanner.close(); scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build(); Assert.assertEquals(ImmutableMap.of(k2, v2, k1, v1), toMap(scanner)); scanner.close(); }