Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:org.apache.accumulo.core.client.rfile.RFileTest.java

License:Apache License

@Test
public void testSampling() throws Exception {

    SortedMap<Key, Value> testData1 = createTestData(1000, 2, 1);

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();

    SamplerConfiguration sc = new SamplerConfiguration(RowSampler.class)
            .setOptions(ImmutableMap.of("hasher", "murmur3_32", "modulus", "19"));

    RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withSampler(sc).build();
    writer.append(testData1.entrySet());
    writer.close();//from  w w  w.ja va  2s.c  o m

    Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build();
    scanner.setSamplerConfiguration(sc);

    RowSampler rowSampler = new RowSampler();
    rowSampler.init(sc);

    SortedMap<Key, Value> sampleData = new TreeMap<>();
    for (Entry<Key, Value> e : testData1.entrySet()) {
        if (rowSampler.accept(e.getKey())) {
            sampleData.put(e.getKey(), e.getValue());
        }
    }

    Assert.assertTrue(sampleData.size() < testData1.size());

    Assert.assertEquals(sampleData, toMap(scanner));

    scanner.clearSamplerConfiguration();

    Assert.assertEquals(testData1, toMap(scanner));

}

---

From source file:org.apache.accumulo.core.client.rfile.RFileTest.java

License:Apache License

@Test
public void testAppendScanner() throws Exception {
    SortedMap<Key, Value> testData = createTestData(10000, 1, 1);
    String testFile = createRFile(testData);

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());

    Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).build();

    String testFile2 = createTmpTestFile();
    RFileWriter writer = RFile.newWriter().to(testFile2).build();
    writer.append(scanner);/* w w w .j  ava 2s  . co m*/
    writer.close();
    scanner.close();

    scanner = RFile.newScanner().from(testFile2).withFileSystem(localFs).build();
    Assert.assertEquals(testData, toMap(scanner));
    scanner.close();
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileTest.java

License:Apache License

@Test
public void testCache() throws Exception {
    SortedMap<Key, Value> testData = createTestData(10000, 1, 1);
    String testFile = createRFile(testData);

    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    Scanner scanner = RFile.newScanner().from(testFile).withFileSystem(localFs).withIndexCache(1000000)
            .withDataCache(10000000).build();

    Random rand = new Random(5);

    for (int i = 0; i < 100; i++) {
        int r = rand.nextInt(10000);
        scanner.setRange(new Range(rowStr(r)));
        Iterator<Entry<Key, Value>> iter = scanner.iterator();
        Assert.assertTrue(iter.hasNext());
        Assert.assertEquals(rowStr(r), iter.next().getKey().getRow().toString());
        Assert.assertFalse(iter.hasNext());
    }//from www.ja  v a  2  s.  co  m

    scanner.close();
}

---

From source file:org.apache.accumulo.core.client.rfile.RFileTest.java

License:Apache License

@Test(expected = IllegalArgumentException.class)
public void testBadVis() throws Exception {
    // this test has two purposes ensure an exception is thrown and ensure the exception document in the javadoc is thrown
    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
    String testFile = createTmpTestFile();
    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) {
        writer.startDefaultLocalityGroup();
        Key k1 = new Key("r1", "f1", "q1", "(A&(B");
        writer.append(k1, new Value("".getBytes()));
    }/*from   ww w  .  j av a 2s.  com*/
}

---

From source file:org.apache.accumulo.core.crypto.CryptoTest.java

License:Apache License

@BeforeClass
public static void setupKeyFiles() throws Exception {
    FileSystem fs = FileSystem.getLocal(hadoopConf);
    Path aesPath = new Path(keyPath);
    try (FSDataOutputStream out = fs.create(aesPath)) {
        out.writeUTF("sixteenbytekey"); // 14 + 2 from writeUTF
    }/*ww  w  .j  ava  2 s.  c  om*/
    try (FSDataOutputStream out = fs.create(new Path(emptyKeyPath))) {
        // auto close after creating
        assertNotNull(out);
    }
}

---

From source file:org.apache.accumulo.core.crypto.CryptoTest.java

License:Apache License

@Test
public void testRFileEncrypted() throws Exception {
    AccumuloConfiguration cryptoOnConf = getAccumuloConfig(CRYPTO_ON_CONF);
    FileSystem fs = FileSystem.getLocal(hadoopConf);
    ArrayList<Key> keys = testData();
    SummarizerConfiguration sumConf = SummarizerConfiguration.builder(KeyCounter.class.getName()).build();

    String file = "target/testFile1.rf";
    fs.delete(new Path(file), true);
    try (RFileWriter writer = RFile.newWriter().to(file).withFileSystem(fs).withTableProperties(cryptoOnConf)
            .withSummarizers(sumConf).build()) {
        Value empty = new Value(new byte[] {});
        writer.startDefaultLocalityGroup();
        for (Key key : keys) {
            writer.append(key, empty);//w  w  w. j a  va 2 s .  c o  m
        }
    }

    Scanner iter = RFile.newScanner().from(file).withFileSystem(fs).withTableProperties(cryptoOnConf).build();
    ArrayList<Key> keysRead = new ArrayList<>();
    iter.forEach(e -> keysRead.add(e.getKey()));
    assertEquals(keys, keysRead);

    Collection<Summary> summaries = RFile.summaries().from(file).withFileSystem(fs)
            .withTableProperties(cryptoOnConf).read();
    Summary summary = Iterables.getOnlyElement(summaries);
    assertEquals(keys.size(), (long) summary.getStatistics().get("keys"));
    assertEquals(1, summary.getStatistics().size());
    assertEquals(0, summary.getFileStatistics().getInaccurate());
    assertEquals(1, summary.getFileStatistics().getTotal());

}

---

From source file:org.apache.accumulo.core.crypto.CryptoTest.java

License:Apache License

@Test
// This test is to ensure when Crypto is configured that it can read unencrypted files
public void testReadNoCryptoWithCryptoConfigured() throws Exception {
    AccumuloConfiguration cryptoOffConf = getAccumuloConfig(CRYPTO_OFF_CONF);
    AccumuloConfiguration cryptoOnConf = getAccumuloConfig(CRYPTO_ON_CONF);
    FileSystem fs = FileSystem.getLocal(hadoopConf);
    ArrayList<Key> keys = testData();

    String file = "target/testFile2.rf";
    fs.delete(new Path(file), true);
    try (RFileWriter writer = RFile.newWriter().to(file).withFileSystem(fs).withTableProperties(cryptoOffConf)
            .build()) {//from  w w w.  ja  va  2  s  .  c  o m
        Value empty = new Value(new byte[] {});
        writer.startDefaultLocalityGroup();
        for (Key key : keys) {
            writer.append(key, empty);
        }
    }

    Scanner iter = RFile.newScanner().from(file).withFileSystem(fs).withTableProperties(cryptoOnConf).build();
    ArrayList<Key> keysRead = new ArrayList<>();
    iter.forEach(e -> keysRead.add(e.getKey()));
    assertEquals(keys, keysRead);
}

---

From source file:org.apache.accumulo.core.file.FileOperationsTest.java

License:Apache License

/**
 * Test for filenames with +1 dot//from   www  .  j av  a 2s . com
 */
@Test
public void handlesFilenamesWithMoreThanOneDot() throws IOException {

    Boolean caughtException = false;
    FileSKVWriter writer = null;
    String filename = "target/test.file." + RFile.EXTENSION;
    File testFile = new File(filename);
    if (testFile.exists()) {
        FileUtils.forceDelete(testFile);
    }
    try {
        FileOperations fileOperations = FileOperations.getInstance();
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.getLocal(conf);
        AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration();
        writer = fileOperations.newWriterBuilder().forFile(filename, fs, conf).withTableConfiguration(acuconf)
                .build();
        writer.close();
    } catch (Exception ex) {
        caughtException = true;
    } finally {
        if (writer != null) {
            writer.close();
        }
        FileUtils.forceDelete(testFile);
    }

    assertFalse("Should not throw with more than 1 dot in filename.", caughtException);
}

---

From source file:org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    FileSystem hadoopFs = FileSystem.get(conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path path = new Path(args[0]);
    FileSystem fs;//from   ww w .  j ava  2s . co  m
    if (args[0].contains(":"))
        fs = path.getFileSystem(conf);
    else
        fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
    printMetaBlockInfo(conf, fs, path);
}

---

From source file:org.apache.accumulo.core.file.rfile.PrintInfo.java

License:Apache License

@Override
public void execute(final String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(PrintInfo.class.getName(), args);
    if (opts.files.isEmpty()) {
        System.err.println("No files were given");
        System.exit(-1);/* w  w w  . j  a  v  a  2  s  .c o  m*/
    }

    Configuration conf = new Configuration();
    for (String confFile : opts.configFiles) {
        log.debug("Adding Hadoop configuration file " + confFile);
        conf.addResource(new Path(confFile));
    }

    FileSystem hadoopFs = FileSystem.get(conf);
    FileSystem localFs = FileSystem.getLocal(conf);

    LogHistogram kvHistogram = new LogHistogram();

    KeyStats dataKeyStats = new KeyStats();
    KeyStats indexKeyStats = new KeyStats();

    for (String arg : opts.files) {
        Path path = new Path(arg);
        FileSystem fs;
        if (arg.contains(":"))
            fs = path.getFileSystem(conf);
        else {
            log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
            fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
        }
        System.out.println(
                "Reading file: " + path.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

        CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null,
                SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
        Reader iter = new RFile.Reader(_rdr);
        MetricsGatherer<Map<String, ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();

        if (opts.vis || opts.hash)
            iter.registerMetrics(vmg);

        iter.printInfo();
        System.out.println();
        org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] { arg });

        Map<String, ArrayList<ByteSequence>> localityGroupCF = null;

        if (opts.histogram || opts.dump || opts.vis || opts.hash || opts.keyStats) {
            localityGroupCF = iter.getLocalityGroupCF();

            FileSKVIterator dataIter;
            if (opts.useSample) {
                dataIter = iter.getSample();

                if (dataIter == null) {
                    System.out.println("ERROR : This rfile has no sample data");
                    return;
                }
            } else {
                dataIter = iter;
            }

            if (opts.keyStats) {
                FileSKVIterator indexIter = iter.getIndex();
                while (indexIter.hasTop()) {
                    indexKeyStats.add(indexIter.getTopKey());
                    indexIter.next();
                }
            }

            for (Entry<String, ArrayList<ByteSequence>> cf : localityGroupCF.entrySet()) {

                dataIter.seek(new Range((Key) null, (Key) null), cf.getValue(), true);
                while (dataIter.hasTop()) {
                    Key key = dataIter.getTopKey();
                    Value value = dataIter.getTopValue();
                    if (opts.dump) {
                        System.out.println(key + " -> " + value);
                        if (System.out.checkError())
                            return;
                    }
                    if (opts.histogram) {
                        kvHistogram.add(key.getSize() + value.getSize());
                    }
                    if (opts.keyStats) {
                        dataKeyStats.add(key);
                    }
                    dataIter.next();
                }
            }
        }

        iter.close();

        if (opts.vis || opts.hash) {
            System.out.println();
            vmg.printMetrics(opts.hash, "Visibility", System.out);
        }

        if (opts.histogram) {
            System.out.println();
            kvHistogram.print("");
        }

        if (opts.keyStats) {
            System.out.println();
            System.out.println("Statistics for keys in data :");
            dataKeyStats.print("\t");
            System.out.println();
            System.out.println("Statistics for keys in index :");
            indexKeyStats.print("\t");
        }
        // If the output stream has closed, there is no reason to keep going.
        if (System.out.checkError())
            return;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »