Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.data.management.copy.CopyableFileUtils.java

License:Apache License

public static CopyableFile createTestCopyableFile(String resourcePath) throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    fs.create(new Path(resourcePath));

    FileStatus status = new FileStatus(0l, false, 0, 0l, 0l, new Path(resourcePath));

    return new CopyableFile(status, new Path(getRandomPath()), null, null, null,
            PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps.<String, String>newHashMap());
}

From source file:gobblin.data.management.copy.extractor.InputStreamExtractorTest.java

License:Apache License

@Test
public void testReadRecord() throws Exception {
    CopyableFile file = getTestCopyableFile("inputStreamExtractorTest/first.txt");

    FileAwareInputStreamExtractor extractor = new FileAwareInputStreamExtractor(
            FileSystem.getLocal(new Configuration()), file);

    FileAwareInputStream fileAwareInputStream = extractor.readRecord(null);

    Assert.assertEquals(fileAwareInputStream.getFile().getOrigin().getPath(), file.getOrigin().getPath());
    Assert.assertEquals(IOUtils.toString(fileAwareInputStream.getInputStream()), "first");

    Assert.assertNull(extractor.readRecord(null));
}

From source file:gobblin.data.management.copy.extractor.InputStreamExtractorTest.java

License:Apache License

private CopyableFile getTestCopyableFile(String resourcePath) throws IOException {
    String filePath = getClass().getClassLoader().getResource(resourcePath).getFile();
    FileStatus status = new FileStatus(0l, false, 0, 0l, 0l, new Path(filePath));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");

    return CopyableFile.fromOriginAndDestination(FileSystem.getLocal(new Configuration()), status,
            new Path("/destination"),
            CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties)
                    .preserve(PreserveAttributes.fromMnemonicString("")).build())
            .build();//from   ww  w.j  a va 2  s . c om
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testDatasetFinder() throws Exception {

    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table2"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table3"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);//from ww  w. ja  v a  2s .com
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 3);
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testException() throws Exception {

    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", TestHiveDatasetFinder.THROW_EXCEPTION));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table3"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);//from  w  w w  . j  a  v  a  2 s . co m
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 2);
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testWhitelist() throws Exception {

    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table2"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db2", "table1"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "db1");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);//from   ww w.  j  a v a2s. com
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 2);
    Assert.assertEquals(datasets.get(0).getTable().getDbName(), "db1");
    Assert.assertEquals(datasets.get(1).getTable().getDbName(), "db1");
    Assert.assertEquals(Sets.newHashSet(datasets.get(0).getTable().getTableName(),
            datasets.get(1).getTable().getTableName()), Sets.newHashSet("table1", "table2"));
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testBlacklist() throws Exception {

    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table2"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db2", "table1"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.BLACKLIST, "db2");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);//from   w w  w  .j a va 2s  .c om
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 2);
    Assert.assertEquals(datasets.get(0).getTable().getDbName(), "db1");
    Assert.assertEquals(datasets.get(1).getTable().getDbName(), "db1");
    Assert.assertEquals(Sets.newHashSet(datasets.get(0).getTable().getTableName(),
            datasets.get(1).getTable().getTableName()), Sets.newHashSet("table1", "table2"));
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testTableList() throws Exception {
    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table2"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table3"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db2", "table1"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.DB_KEY, "db1");
    properties.put(HiveDatasetFinder.TABLE_PATTERN_KEY, "table1|table2");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);//from  w  w  w  .  j  a v a 2 s .c  o m
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 2);
    Assert.assertEquals(datasets.get(0).getTable().getDbName(), "db1");
    Assert.assertEquals(datasets.get(1).getTable().getDbName(), "db1");
    Assert.assertEquals(Sets.newHashSet(datasets.get(0).getTable().getTableName(),
            datasets.get(1).getTable().getTableName()), Sets.newHashSet("table1", "table2"));
}

From source file:gobblin.data.management.copy.hive.HiveDatasetFinderTest.java

License:Apache License

@Test
public void testDatasetConfig() throws Exception {

    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);

    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");

    properties.put("hive.dataset.test.conf1", "conf1-val1");
    properties.put("hive.dataset.test.conf2", "conf2-val2");

    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties,
            pool);/*from  w  w  w.j  av  a2  s.c o  m*/
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 1);
    HiveDataset hiveDataset = datasets.get(0);

    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf1"), "conf1-val1");
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf2"), "conf2-val2");

    // Test scoped configs with prefix
    properties.put(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, "hive.dataset.test");

    finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool);
    datasets = Lists.newArrayList(finder.getDatasetsIterator());

    Assert.assertEquals(datasets.size(), 1);
    hiveDataset = datasets.get(0);
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf1"), "conf1-val1");
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf2"), "conf2-val2");

}

From source file:gobblin.data.management.copy.predicates.RegistrationTimeSkipPredicateTest.java

License:Apache License

@Test
public void test() throws Exception {

    Path partition1Path = new Path("/path/to/partition1");
    long modTime = 100000;

    CopyContext copyContext = new CopyContext();
    CopyConfiguration copyConfiguration = Mockito.mock(CopyConfiguration.class);
    Mockito.doReturn(copyContext).when(copyConfiguration).getCopyContext();
    HiveDataset dataset = Mockito.mock(HiveDataset.class);
    FileSystem fs = Mockito.spy(FileSystem.getLocal(new Configuration()));
    FileStatus status = new FileStatus(1, false, 1, 1, modTime, partition1Path);
    Path qualifiedPath = fs.makeQualified(partition1Path);
    Mockito.doReturn(status).when(fs).getFileStatus(qualifiedPath);
    Mockito.doReturn(status).when(fs).getFileStatus(partition1Path);
    Mockito.doReturn(fs).when(dataset).getFs();

    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
    Mockito.doReturn(copyConfiguration).when(helper).getConfiguration();
    Mockito.doReturn(dataset).when(helper).getDataset();

    RegistrationTimeSkipPredicate predicate = new RegistrationTimeSkipPredicate(helper);

    // partition exists, but registration time before modtime => don't skip
    HivePartitionFileSet pc = createPartitionCopy(partition1Path, modTime - 1, true);
    Assert.assertFalse(predicate.apply(pc));

    // partition exists, registration time equal modtime => don't skip
    pc = createPartitionCopy(partition1Path, modTime, true);
    Assert.assertFalse(predicate.apply(pc));

    // partition exists, registration time larger modtime => do skip
    pc = createPartitionCopy(partition1Path, modTime + 1, true);
    Assert.assertTrue(predicate.apply(pc));

    // partition doesn't exist => don't skip
    pc = createPartitionCopy(partition1Path, modTime + 1, false);
    Assert.assertFalse(predicate.apply(pc));

    // partition exists but is not annotated => don't skip
    pc = createPartitionCopy(partition1Path, modTime + 1, true);
    pc.getExistingTargetPartition().get().getParameters().clear();
    Assert.assertFalse(predicate.apply(pc));

}