Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:org.apache.accumulo.core.volume.NonConfiguredVolumeTest.java

License:Apache License

@Before
public void create() throws IOException {
    volume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
}

From source file:org.apache.accumulo.core.volume.NonConfiguredVolumeTest.java

License:Apache License

@Test
public void testSameFileSystem() throws IOException {
    Assert.assertEquals(FileSystem.getLocal(new Configuration()), volume.getFileSystem());
}

From source file:org.apache.accumulo.core.volume.NonConfiguredVolumeTest.java

License:Apache License

@Test
public void testEquality() throws IOException {
    Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
    Assert.assertEquals(volume, newVolume);
}

From source file:org.apache.accumulo.core.volume.NonConfiguredVolumeTest.java

License:Apache License

@Test
public void testHashCode() throws IOException {
    Volume newVolume = new NonConfiguredVolume(FileSystem.getLocal(new Configuration()));
    Assert.assertEquals(volume.hashCode(), newVolume.hashCode());
}

From source file:org.apache.accumulo.hadoop.its.mapred.AccumuloFileOutputFormatIT.java

License:Apache License

private void handleWriteTests(boolean content) throws Exception {
    File f = folder.newFile(testName.getMethodName());
    if (f.delete()) {
        log.debug("Deleted {}", f);
    }/*from   w  w w .ja va2  s . c om*/
    MRTester.main(new String[] { content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath() });

    assertTrue(f.exists());
    File[] files = f.listFiles(file -> file.getName().startsWith("part-m-"));
    assertNotNull(files);
    if (content) {
        assertEquals(1, files.length);
        assertTrue(files[0].exists());

        Configuration conf = cluster.getServerContext().getHadoopConf();
        DefaultConfiguration acuconf = DefaultConfiguration.getInstance();
        FileSKVIterator sample = FileOperations.getInstance().newReaderBuilder()
                .forFile(files[0].toString(), FileSystem.getLocal(conf), conf,
                        CryptoServiceFactory.newDefaultInstance())
                .withTableConfiguration(acuconf).build()
                .getSample(new SamplerConfigurationImpl(SAMPLER_CONFIG));
        assertNotNull(sample);
    } else {
        assertEquals(0, files.length);
    }
}

From source file:org.apache.accumulo.hadoop.its.mapreduce.AccumuloFileOutputFormatIT.java

License:Apache License

private void handleWriteTests(boolean content) throws Exception {
    File f = folder.newFile(testName.getMethodName());
    assertTrue(f.delete());/*from   w w  w  .  java2s . co  m*/
    MRTester.main(new String[] { content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath() });

    assertTrue(f.exists());
    File[] files = f.listFiles(file -> file.getName().startsWith("part-m-"));
    assertNotNull(files);
    if (content) {
        assertEquals(1, files.length);
        assertTrue(files[0].exists());

        Configuration conf = cluster.getServerContext().getHadoopConf();
        DefaultConfiguration acuconf = DefaultConfiguration.getInstance();
        FileSKVIterator sample = RFileOperations.getInstance().newReaderBuilder()
                .forFile(files[0].toString(), FileSystem.getLocal(conf), conf,
                        CryptoServiceFactory.newDefaultInstance())
                .withTableConfiguration(acuconf).build()
                .getSample(new SamplerConfigurationImpl(SAMPLER_CONFIG));
        assertNotNull(sample);
    } else {
        assertEquals(0, files.length);
    }
}

From source file:org.apache.accumulo.server.client.BulkImporterTest.java

License:Apache License

@Test
public void testFindOverlappingTablets() throws Exception {
    MockTabletLocator locator = new MockTabletLocator();
    FileSystem fs = FileSystem.getLocal(CachedConfiguration.getInstance());
    ClientContext context = EasyMock.createMock(ClientContext.class);
    EasyMock.expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
    EasyMock.replay(context);/*from ww w . ja  v a  2s .co  m*/
    String file = "target/testFile.rf";
    fs.delete(new Path(file), true);
    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(file, fs, fs.getConf())
            .withTableConfiguration(context.getConfiguration()).build();
    writer.startDefaultLocalityGroup();
    Value empty = new Value(new byte[] {});
    writer.append(new Key("a", "cf", "cq"), empty);
    writer.append(new Key("a", "cf", "cq1"), empty);
    writer.append(new Key("a", "cf", "cq2"), empty);
    writer.append(new Key("a", "cf", "cq3"), empty);
    writer.append(new Key("a", "cf", "cq4"), empty);
    writer.append(new Key("a", "cf", "cq5"), empty);
    writer.append(new Key("d", "cf", "cq"), empty);
    writer.append(new Key("d", "cf", "cq1"), empty);
    writer.append(new Key("d", "cf", "cq2"), empty);
    writer.append(new Key("d", "cf", "cq3"), empty);
    writer.append(new Key("d", "cf", "cq4"), empty);
    writer.append(new Key("d", "cf", "cq5"), empty);
    writer.append(new Key("dd", "cf", "cq1"), empty);
    writer.append(new Key("ichabod", "cf", "cq"), empty);
    writer.append(new Key("icky", "cf", "cq1"), empty);
    writer.append(new Key("iffy", "cf", "cq2"), empty);
    writer.append(new Key("internal", "cf", "cq3"), empty);
    writer.append(new Key("is", "cf", "cq4"), empty);
    writer.append(new Key("iterator", "cf", "cq5"), empty);
    writer.append(new Key("xyzzy", "cf", "cq"), empty);
    writer.close();
    VolumeManager vm = VolumeManagerImpl.get(context.getConfiguration());
    List<TabletLocation> overlaps = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file));
    Assert.assertEquals(5, overlaps.size());
    Collections.sort(overlaps);
    Assert.assertEquals(new KeyExtent(tableId, new Text("a"), null), overlaps.get(0).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps.get(1).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps.get(2).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps.get(3).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, null, new Text("l")), overlaps.get(4).tablet_extent);

    List<TabletLocation> overlaps2 = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file),
            new KeyExtent(tableId, new Text("h"), new Text("b")));
    Assert.assertEquals(3, overlaps2.size());
    Assert.assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps2.get(0).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps2.get(1).tablet_extent);
    Assert.assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps2.get(2).tablet_extent);
    Assert.assertEquals(locator.invalidated, 1);
}

From source file:org.apache.accumulo.server.fs.VolumeManagerImpl.java

License:Apache License

public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
    AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
    Volume defaultLocalVolume = VolumeConfiguration
            .create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);

    // The default volume gets placed in the map, but local filesystem is only used for testing purposes
    return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume,
            accConf);/*ww w .j  a  va  2 s  .  co  m*/
}

From source file:org.apache.accumulo.server.fs.VolumeUtilTest.java

License:Apache License

@Test
public void testSame() throws Exception {
    FileSystem fs = FileSystem.getLocal(new Configuration());

    Path subdir1 = new Path(tempFolder.newFolder().toURI());
    Path subdir2 = new Path(tempFolder.newFolder().toURI());
    Path subdir3 = new Path(tempFolder.newFolder().toURI());

    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs,
            new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497")));
    Assert.assertFalse(VolumeUtil.same(fs,
            new Path(tempFolder.getRoot().toURI().toString(), "8854339269459287524098238497"), fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));

    writeFile(fs, subdir1, "abc", "foo");
    writeFile(fs, subdir2, "abc", "bar");
    writeFile(fs, subdir3, "abc", "foo");

    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));
    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir2));
    Assert.assertFalse(VolumeUtil.same(fs, subdir2, fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertTrue(VolumeUtil.same(fs, subdir3, fs, subdir1));

    writeFile(fs, subdir1, "def", "123456");
    writeFile(fs, subdir2, "def", "123456");
    writeFile(fs, subdir3, "def", "123456");

    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir1));
    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir2));
    Assert.assertFalse(VolumeUtil.same(fs, subdir2, fs, subdir1));
    Assert.assertTrue(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertTrue(VolumeUtil.same(fs, subdir3, fs, subdir1));

    writeFile(fs, subdir3, "ghi", "09876");

    Assert.assertFalse(VolumeUtil.same(fs, subdir1, fs, subdir3));
    Assert.assertFalse(VolumeUtil.same(fs, subdir3, fs, subdir1));

    fs.mkdirs(new Path(subdir2, "dir1"));

    try {//from www .  j  a v  a  2  s. c  o  m
        VolumeUtil.same(fs, subdir1, fs, subdir2);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, subdir2, fs, subdir1);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, subdir1, fs, new Path(subdir2, "def"));
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

    try {
        VolumeUtil.same(fs, new Path(subdir2, "def"), fs, subdir3);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }

}

From source file:org.apache.accumulo.server.logger.LogWriter.java

License:Apache License

/**
 * //from  w  w  w. j a va  2  s  .  c  o m
 * @param fs
 *          The HDFS instance shared by master/tservers.
 * @param logDirectory
 *          The local directory to write the recovery logs.
 * @param instanceId
 *          The accumulo instance for which we are logging.
 */
public LogWriter(AccumuloConfiguration acuConf, FileSystem fs, Collection<String> logDirectories,
        String instanceId, int threadPoolSize, boolean archive) {
    this.acuConf = acuConf;
    this.fs = fs;
    this.roots = new ArrayList<String>(logDirectories);
    this.instanceId = instanceId;
    this.copyThreadPool = Executors.newFixedThreadPool(threadPoolSize);
    try {
        this.logArchiver = new LogArchiver(acuConf, TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf())), fs,
                new ArrayList<String>(logDirectories), archive);
    } catch (IOException e1) {
        throw new RuntimeException(e1);
    }

    // Register the metrics MBean
    try {
        metrics.register();
    } catch (Exception e) {
        log.error("Exception registering MBean with MBean Server", e);
    }
}