Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test
public void testSummaries() throws Exception {
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(VisibilitySummarizer.class).build();
    SummarizerConfiguration sc2 = SummarizerConfiguration.builder(FamilySummarizer.class).build();

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();

    SortedMap<Key, Value> testData1 = createTestData(0, 100, 0, 4, 1, "A&B", "A&B&C");

    RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withSummarizers(sc1, sc2)
            .build();/*from www .  ja  v  a  2  s  .com*/
    writer.append(testData1.entrySet());
    writer.close();

    // verify summary data
    Collection<Summary> summaries = RFile.summaries().from(testFile).withFileSystem(localFs).read();
    assertEquals(2, summaries.size());
    for (Summary summary : summaries) {
        assertEquals(0, summary.getFileStatistics().getInaccurate());
        assertEquals(1, summary.getFileStatistics().getTotal());
        String className = summary.getSummarizerConfiguration().getClassName();
        CounterSummary counterSummary = new CounterSummary(summary);
        if (className.equals(FamilySummarizer.class.getName())) {
            Map<String, Long> counters = counterSummary.getCounters();
            Map<String, Long> expected = ImmutableMap.of("0000", 200L, "0001", 200L, "0002", 200L, "0003",
                    200L);
            assertEquals(expected, counters);
        } else if (className.equals(VisibilitySummarizer.class.getName())) {
            Map<String, Long> counters = counterSummary.getCounters();
            Map<String, Long> expected = ImmutableMap.of("A&B", 400L, "A&B&C", 400L);
            assertEquals(expected, counters);
        } else {
            fail("Unexpected classname " + className);
        }
    }

    // check if writing summary data impacted normal rfile functionality
    Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs)
            .withAuthorizations(new Authorizations("A", "B", "C")).build();
    assertEquals(testData1, toMap(scanner));
    scanner.close();

    String testFile2 = createTmpTestFile();
    SortedMap<Key, Value> testData2 = createTestData(100, 100, 0, 4, 1, "A&B", "A&B&C");
    writer = RFile.newWriter().to(testFile2).withFileSystem(localFs).withSummarizers(sc1, sc2).build();
    writer.append(testData2.entrySet());
    writer.close();

    // verify reading summaries from multiple files works
    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs).read();
    assertEquals(2, summaries.size());
    for (Summary summary : summaries) {
        assertEquals(0, summary.getFileStatistics().getInaccurate());
        assertEquals(2, summary.getFileStatistics().getTotal());
        String className = summary.getSummarizerConfiguration().getClassName();
        CounterSummary counterSummary = new CounterSummary(summary);
        if (className.equals(FamilySummarizer.class.getName())) {
            Map<String, Long> counters = counterSummary.getCounters();
            Map<String, Long> expected = ImmutableMap.of("0000", 400L, "0001", 400L, "0002", 400L, "0003",
                    400L);
            assertEquals(expected, counters);
        } else if (className.equals(VisibilitySummarizer.class.getName())) {
            Map<String, Long> counters = counterSummary.getCounters();
            Map<String, Long> expected = ImmutableMap.of("A&B", 800L, "A&B&C", 800L);
            assertEquals(expected, counters);
        } else {
            fail("Unexpected classname " + className);
        }
    }

    // verify reading a subset of summaries works
    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 0);

    // the following test check boundary conditions for start row and end row
    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(99)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 400L, "A&B&C", 400L), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(98)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(0)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow("#").read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(100)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 400L, "A&B&C", 400L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).endRow(rowStr(99)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 400L, "A&B&C", 400L), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).endRow(rowStr(100)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).endRow(rowStr(199)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(50)).endRow(rowStr(150)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 2);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(120)).endRow(rowStr(150)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 400L, "A&B&C", 400L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(50)).endRow(rowStr(199)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow("#").endRow(rowStr(150)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 800L, "A&B&C", 800L), 1);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(199)).read();
    checkSummaries(summaries, ImmutableMap.of(), 0);
    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).startRow(rowStr(200)).read();
    checkSummaries(summaries, ImmutableMap.of(), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).endRow("#").read();
    checkSummaries(summaries, ImmutableMap.of(), 0);

    summaries = RFile.summaries().from(testFile, testFile2).withFileSystem(localFs)
            .selectSummaries(sc -> sc.equals(sc1)).endRow(rowStr(0)).read();
    checkSummaries(summaries, ImmutableMap.of("A&B", 400L, "A&B&C", 400L), 1);
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testOutOfOrder() throws Exception {
    // test that exception declared in API is thrown
    Key k1 = new Key("r1", "f1", "q1");
    Value v1 = new Value("1".getBytes());

    Key k2 = new Key("r2", "f1", "q1");
    Value v2 = new Value("2".getBytes());

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.append(k2, v2);/*ww w  .j  a v  a 2 s. c o m*/
        writer.append(k1, v1);
    }
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testOutOfOrderIterable() throws Exception {
    // test that exception declared in API is thrown
    Key k1 = new Key("r1", "f1", "q1");
    Value v1 = new Value("1".getBytes());

    Key k2 = new Key("r2", "f1", "q1");
    Value v2 = new Value("2".getBytes());

    ArrayList<Entry<Key, Value>> data = new ArrayList<>();
    data.add(new AbstractMap.SimpleEntry<>(k2, v2));
    data.add(new AbstractMap.SimpleEntry<>(k1, v1));

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.append(data);//from w ww  .  j a  v  a  2s .  c o  m
    }
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testBadVis() throws Exception {
    // this test has two purposes ensure an exception is thrown and ensure the exception document in
    // the javadoc is thrown
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startDefaultLocalityGroup();
        Key k1 = new Key("r1", "f1", "q1", "(A&(B");
        writer.append(k1, new Value("".getBytes()));
    }//w ww.ja  va2  s  . c  om
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testBadVisIterable() throws Exception {
    // test append(iterable) method
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startDefaultLocalityGroup();
        Key k1 = new Key("r1", "f1", "q1", "(A&(B");
        Entry<Key, Value> entry = new AbstractMap.SimpleEntry<>(k1, new Value("".getBytes()));
        writer.append(Collections.singletonList(entry));
    }//w ww  .  j av a2  s .  co  m
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalStateException.class)
public void testDoubleStart() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startDefaultLocalityGroup();
        writer.startDefaultLocalityGroup();
    }//  w  w  w .  j  a v a2  s.  co m
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalStateException.class)
public void testAppendStartDefault() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.append(new Key("r1", "f1", "q1"), new Value("1".getBytes()));
        writer.startDefaultLocalityGroup();
    }//  w  ww . j a va  2  s .co  m
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalStateException.class)
public void testStartAfter() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        Key k1 = new Key("r1", "f1", "q1");
        writer.append(k1, new Value("".getBytes()));
        writer.startNewLocalityGroup("lg1", "fam1");
    }//from  ww  w .java  2 s . c  o  m
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testIllegalColumn() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startNewLocalityGroup("lg1", "fam1");
        Key k1 = new Key("r1", "f1", "q1");
        // should not be able to append the column family f1
        writer.append(k1, new Value("".getBytes()));
    }/* w  ww  .  ja va  2  s.  c  om*/
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileClientTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testWrongGroup() throws Exception {
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startNewLocalityGroup("lg1", "fam1");
        Key k1 = new Key("r1", "fam1", "q1");
        writer.append(k1, new Value("".getBytes()));
        writer.startDefaultLocalityGroup();
        // should not be able to append the column family fam1 to default locality group
        Key k2 = new Key("r1", "fam1", "q2");
        writer.append(k2, new Value("".getBytes()));
    }/*  w  ww  .  j  av a  2 s.  c  o m*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »