Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.compaction.mapreduce.MRCompactor.java

License:Apache License

/**
 * Copy dependency jars from local fs to HDFS.
 *//*  w w w.j ava 2 s.  co  m*/
private void copyDependencyJarsToHdfs() throws IOException {
    if (!this.state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) {
        return;
    }
    LocalFileSystem lfs = FileSystem.getLocal(this.conf);
    Path tmpJarFileDir = new Path(this.tmpOutputDir, "_gobblin_compaction_jars");
    this.state.setProp(COMPACTION_JARS, tmpJarFileDir.toString());
    this.fs.delete(tmpJarFileDir, true);
    for (String jarFile : this.state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) {
        for (FileStatus status : lfs.globStatus(new Path(jarFile))) {
            Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName());
            this.fs.copyFromLocalFile(status.getPath(), tmpJarFile);
            LOG.info(String.format("%s will be added to classpath", tmpJarFile));
        }
    }
}

From source file:gobblin.compaction.mapreduce.MRCompactorJobRunnerFilenameRecordCountProviderTest.java

License:Apache License

@Test
public void testFileNameRecordCountProvider() throws IOException {
    String originalFilename = "test.123.avro";
    String suffixPattern = Pattern.quote(".late") + "[\\d]*";

    Path testDir = new Path("/tmp/compactorFilenameRecordCountProviderTest");
    FileSystem fs = FileSystem.getLocal(new Configuration());
    try {/*from   w  w w  .  ja  va  2s.c o m*/
        if (fs.exists(testDir)) {
            fs.delete(testDir, true);
        }
        fs.mkdirs(testDir);

        RecordCountProvider originFileNameFormat = new IngestionRecordCountProvider();

        LateFileRecordCountProvider lateFileRecordCountProvider = new LateFileRecordCountProvider(
                originFileNameFormat);

        Path firstOutput = lateFileRecordCountProvider.constructLateFilePath(originalFilename, fs, testDir);

        Assert.assertEquals(new Path(testDir, originalFilename), firstOutput);
        Assert.assertEquals(123, lateFileRecordCountProvider.getRecordCount(firstOutput));

        fs.create(firstOutput);
        Pattern pattern1 = Pattern.compile(
                Pattern.quote(Files.getNameWithoutExtension(originalFilename)) + suffixPattern + "\\.avro");
        Path secondOutput = lateFileRecordCountProvider.constructLateFilePath(firstOutput.getName(), fs,
                testDir);
        Assert.assertEquals(testDir, secondOutput.getParent());
        Assert.assertTrue(pattern1.matcher(secondOutput.getName()).matches());
        Assert.assertEquals(123, lateFileRecordCountProvider.getRecordCount(secondOutput));

        fs.create(secondOutput);
        Pattern pattern2 = Pattern.compile(
                Files.getNameWithoutExtension(originalFilename) + suffixPattern + suffixPattern + "\\.avro");
        Path thirdOutput = lateFileRecordCountProvider.constructLateFilePath(secondOutput.getName(), fs,
                testDir);
        Assert.assertEquals(testDir, thirdOutput.getParent());
        Assert.assertTrue(pattern2.matcher(thirdOutput.getName()).matches());
        Assert.assertEquals(123, lateFileRecordCountProvider.getRecordCount(thirdOutput));
    } finally {
        fs.delete(testDir, true);
    }
}

From source file:gobblin.config.store.hdfs.SimpleHdfsConfigStoreTest.java

License:Apache License

@BeforeClass
public void setUp() throws URISyntaxException, ConfigStoreCreationException, IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.fs.mkdirs(CONFIG_DIR_PATH);

    SimpleLocalHDFSConfigStoreFactory simpleHDFSConfigStoreConfigFactory = new SimpleLocalHDFSConfigStoreFactory();

    URI storeURI = getStoreURI(System.getProperty("user.dir") + File.separator + CONFIG_DIR_NAME);
    this.simpleHDFSConfigStore = simpleHDFSConfigStoreConfigFactory.createConfigStore(storeURI);
    this.simpleHDFSConfigStore
            .deploy(new FsDeploymentConfig(new ClasspathConfigSource(new Properties()), VERSION));

}

From source file:gobblin.config.store.hdfs.SimpleHdfsConfigureStoreFactoryTest.java

License:Apache License

@Test
public void testGetDefaults() throws URISyntaxException, ConfigStoreCreationException, IOException {
    Path configStoreDir = new Path(SimpleHDFSConfigStore.CONFIG_STORE_NAME);
    FileSystem localFS = FileSystem.getLocal(new Configuration());

    try {/*ww  w. j a v  a2 s  .co  m*/
        Assert.assertTrue(localFS.mkdirs(configStoreDir));

        SimpleLocalHDFSConfigStoreFactory simpleLocalHDFSConfigStoreFactory = new SimpleLocalHDFSConfigStoreFactory();

        URI configKey = new URI(simpleLocalHDFSConfigStoreFactory.getScheme(), "", "", "", "");
        SimpleHDFSConfigStore simpleHDFSConfigStore = simpleLocalHDFSConfigStoreFactory
                .createConfigStore(configKey);

        Assert.assertEquals(simpleHDFSConfigStore.getStoreURI().getScheme(),
                simpleLocalHDFSConfigStoreFactory.getScheme());
        Assert.assertNull(simpleHDFSConfigStore.getStoreURI().getAuthority());
        Assert.assertEquals(simpleHDFSConfigStore.getStoreURI().getPath(), System.getProperty("user.dir"));
    } finally {
        localFS.delete(configStoreDir, true);
    }
}

From source file:gobblin.config.store.hdfs.SimpleHdfsConfigureStoreFactoryTest.java

License:Apache License

@Test
public void testConfiguration() throws Exception {
    FileSystem localFS = FileSystem.getLocal(new Configuration());
    Path testRoot = localFS.makeQualified(new Path("testConfiguration"));
    Path configRoot = localFS.makeQualified(new Path(testRoot, "dir2"));
    Path configStoreRoot = new Path(configRoot, SimpleHDFSConfigStore.CONFIG_STORE_NAME);
    Assert.assertTrue(localFS.mkdirs(configStoreRoot));
    try {/*  w w w  . ja v  a  2 s  . co  m*/
        Config confConf1 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(configRoot.toString()));
        SimpleHDFSConfigStoreFactory confFactory = new SimpleHDFSConfigStoreFactory(confConf1);
        Assert.assertTrue(confFactory.hasDefaultStoreURI());
        Assert.assertEquals(confFactory.getDefaultStoreURI(), configRoot.toUri());
        Assert.assertEquals(confFactory.getPhysicalScheme(), "file");
        Assert.assertEquals(confFactory.getDefaultRootDir().toString(),
                "file:" + System.getProperty("user.home"));

        // Valid path
        SimpleHDFSConfigStore store1 = confFactory.createConfigStore(new URI("simple-file:/d"));
        Assert.assertEquals(store1.getStoreURI().getScheme(), confFactory.getScheme());
        Assert.assertEquals(store1.getStoreURI().getAuthority(),
                confFactory.getDefaultStoreURI().getAuthority());
        Assert.assertEquals(store1.getStoreURI().getPath(), confFactory.getDefaultStoreURI().getPath());

        // Invalid path
        Config confConf2 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(testRoot.toString()));
        try {
            new SimpleHDFSConfigStoreFactory(confConf2);
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("Path does not appear to be a config store root"));
        }

        // Empty path
        Config confConf3 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(""));
        try {
            new SimpleHDFSConfigStoreFactory(confConf3);
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("Default store URI should be non-empty"));
        }
    } finally {
        localFS.delete(testRoot, true);
    }
}

From source file:gobblin.data.management.copy.converter.DecryptConverterTest.java

License:Apache License

@Test
public void testConvertGpgRecord() throws Exception {
    final String expectedFileContents = "123456789";
    final String passphrase = "12";
    DecryptConverter converter = new DecryptConverter();
    WorkUnitState workUnitState = new WorkUnitState();

    try {/*from  www . j a va 2  s.  c  om*/
        setEncryptedPassphrase(passphrase, workUnitState);
        converter.init(workUnitState);

        FileSystem fs = FileSystem.getLocal(new Configuration());

        URL url = getClass().getClassLoader().getResource("decryptConverterTest/decrypt-test.txt.gpg");
        Assert.assertNotNull(url);

        String gpgFilePath = url.getFile();
        try (FSDataInputStream gpgFileInput = fs.open(new Path(gpgFilePath))) {
            FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(
                    CopyableFileUtils.getTestCopyableFile(), gpgFileInput);

            Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema",
                    fileAwareInputStream, workUnitState);
            fileAwareInputStream = Iterables.getFirst(iterable, null);
            Assert.assertNotNull(fileAwareInputStream);

            String actual = IOUtils.toString(fileAwareInputStream.getInputStream(), Charsets.UTF_8);
            Assert.assertEquals(actual, expectedFileContents);
        }
    } finally {
        deleteMasterPwdFile();
        converter.close();
    }
}

From source file:gobblin.data.management.copy.converter.DecryptConverterTest.java

License:Apache License

@Test
public void testConvertDifferentEncryption() throws IOException, DataConversionException {
    final String expectedFileContents = "2345678";

    WorkUnitState workUnitState = new WorkUnitState();
    workUnitState.getJobState().setProp("converter.encrypt." + EncryptionConfigParser.ENCRYPTION_ALGORITHM_KEY,
            "insecure_shift");

    try (DecryptConverter converter = new DecryptConverter()) {
        converter.init(workUnitState);/* ww w .  j  ava 2s  . c  o  m*/
        FileSystem fs = FileSystem.getLocal(new Configuration());

        URL url = getClass().getClassLoader()
                .getResource("decryptConverterTest/decrypt-test.txt.insecure_shift");
        Assert.assertNotNull(url);

        String testFilePath = url.getFile();
        try (FSDataInputStream testFileInput = fs.open(new Path(testFilePath))) {
            FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(
                    CopyableFileUtils.getTestCopyableFile(), testFileInput);
            fileAwareInputStream.getFile()
                    .setDestination(new Path("file:///tmp/decrypt-test.txt.insecure_shift"));
            Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema",
                    fileAwareInputStream, workUnitState);
            FileAwareInputStream decryptedStream = Iterables.getFirst(iterable, null);
            Assert.assertNotNull(decryptedStream);

            String actual = IOUtils.toString(decryptedStream.getInputStream(), Charsets.UTF_8);
            Assert.assertEquals(actual, expectedFileContents);
            Assert.assertEquals(decryptedStream.getFile().getDestination().getName(), "decrypt-test.txt");
        }
    }
}

From source file:gobblin.data.management.copy.converter.UnGzipConverterTest.java

License:Apache License

@Test(dataProvider = "fileDataProvider")
public void testGz(final String filePath, final String expectedText) throws Exception {

    UnGzipConverter converter = new UnGzipConverter();

    FileSystem fs = FileSystem.getLocal(new Configuration());

    String fullPath = getClass().getClassLoader().getResource(filePath).getFile();
    FileAwareInputStream fileAwareInputStream = new FileAwareInputStream(
            CopyableFileUtils.getTestCopyableFile(filePath), fs.open(new Path(fullPath)));

    Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema", fileAwareInputStream,
            new WorkUnitState());

    String actual = readGzipStreamAsString(Iterables.getFirst(iterable, null).getInputStream());
    Assert.assertEquals(actual.trim(), expectedText);

}

From source file:gobblin.data.management.copy.CopyableFileTest.java

License:Apache License

@Test
public void testCopyableFileBuilderMinimumConfiguration() throws IOException {
    // Source/*from  w w  w  .java2  s. c om*/
    String datasetRootDir = "/data/databases/source";
    Path datasetRoot = new Path(datasetRootDir);
    FileSystem originFS = FileSystem.getLocal(new Configuration());
    Path originFile = new Path(datasetRootDir, "copyableFile");
    FileStatus origin = new FileStatus(0l, false, 0, 0l, System.currentTimeMillis(), originFile);
    PreserveAttributes preserveAttributes = PreserveAttributes.fromMnemonicString("ugp");

    // Target
    String targetRoot = "/data/databases/destination";
    Path relativePath = PathUtils.relativizePath(originFile, datasetRoot);
    Path targetPath = new Path(targetRoot, relativePath);

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
    CopyConfiguration copyConfiguration = CopyConfiguration
            .builder(FileSystem.getLocal(new Configuration()), properties).preserve(preserveAttributes).build();

    CopyableFile copyableFile = CopyableFile.builder(originFS, origin, datasetRoot, copyConfiguration)
            .destination(targetPath).ancestorsOwnerAndPermission(Lists.<OwnerAndPermission>newArrayList()) // not testing ancestors
            .build();

    // Making sure all fields are populated correctly via CopyableFile builder

    // Verify preserve attribute options
    Assert.assertEquals(copyableFile.getPreserve().toMnemonicString(), preserveAttributes.toMnemonicString());

    // Verify origin
    Assert.assertEquals(copyableFile.getFileSet(), "");
    Assert.assertEquals(copyableFile.getOrigin(), origin);

    // Verify destination target, permissions and other attributes
    Assert.assertEquals(copyableFile.getChecksum().length, 0);
    Assert.assertEquals(copyableFile.getDestination().toString(), targetPath.toString());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getGroup(), origin.getGroup());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getOwner(), origin.getOwner());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getFsPermission(),
            origin.getPermission());

    // Verify auto determined timestamp
    Assert.assertEquals(copyableFile.getOriginTimestamp(), origin.getModificationTime());
    Assert.assertEquals(copyableFile.getUpstreamTimestamp(), origin.getModificationTime());
}

From source file:gobblin.data.management.copy.CopyableFileTest.java

License:Apache License

@Test
public void testCopyableFileBuilderMaximumConfiguration() throws IOException {
    // Source//from w  ww . j  a v a2  s.  co m
    String datasetRootDir = "/data/databases/source";
    Path datasetRoot = new Path(datasetRootDir);
    FileSystem originFS = FileSystem.getLocal(new Configuration());
    Path originFile = new Path(datasetRootDir, "copyableFile");
    FileStatus origin = new FileStatus(0l, false, 0, 0l, System.currentTimeMillis(), originFile);
    PreserveAttributes preserveAttributes = PreserveAttributes.fromMnemonicString("ugp");

    // Target
    String targetRoot = "/data/databases/destination";
    Path relativePath = PathUtils.relativizePath(originFile, datasetRoot);
    Path targetPath = new Path(targetRoot, relativePath);

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
    CopyConfiguration copyConfiguration = CopyConfiguration
            .builder(FileSystem.getLocal(new Configuration()), properties).preserve(preserveAttributes).build();

    // Other attributes
    String fileSet = "fileset";
    byte[] checksum = new byte[1];
    long originTimestamp = 23091986l;
    long upstreamTimestamp = 23091986l;
    OwnerAndPermission ownerAndPermission = new OwnerAndPermission("gobblin", "gobblin-dev",
            origin.getPermission());

    CopyableFile copyableFile = CopyableFile.builder(originFS, origin, datasetRoot, copyConfiguration)
            .fileSet(fileSet).checksum(checksum).originTimestamp(originTimestamp)
            .upstreamTimestamp(upstreamTimestamp).destinationOwnerAndPermission(ownerAndPermission)
            .origin(origin).preserve(preserveAttributes).destination(targetPath)
            .ancestorsOwnerAndPermission(Lists.<OwnerAndPermission>newArrayList()).build();

    // Verify preserve attribute options
    Assert.assertEquals(copyableFile.getPreserve().toMnemonicString(), preserveAttributes.toMnemonicString());

    // Verify origin
    Assert.assertEquals(copyableFile.getFileSet(), fileSet);
    Assert.assertEquals(copyableFile.getOrigin(), origin);

    // Verify destination target, permissions and other attributes
    Assert.assertEquals(copyableFile.getChecksum().length, 1);
    Assert.assertEquals(copyableFile.getDestination().toString(), targetPath.toString());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getGroup(),
            ownerAndPermission.getGroup());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getOwner(),
            ownerAndPermission.getOwner());
    Assert.assertEquals(copyableFile.getDestinationOwnerAndPermission().getFsPermission(),
            ownerAndPermission.getFsPermission());

    // Verify auto determined timestamp
    Assert.assertEquals(copyableFile.getOriginTimestamp(), originTimestamp);
    Assert.assertEquals(copyableFile.getUpstreamTimestamp(), upstreamTimestamp);
}