Example usage for org.apache.hadoop.fs FileSystem mkdirs

List of usage examples for org.apache.hadoop.fs FileSystem mkdirs

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:gobblin.config.store.hdfs.SimpleHdfsConfigureStoreFactoryTest.java

License:Apache License

@Test
public void testConfiguration() throws Exception {
    FileSystem localFS = FileSystem.getLocal(new Configuration());
    Path testRoot = localFS.makeQualified(new Path("testConfiguration"));
    Path configRoot = localFS.makeQualified(new Path(testRoot, "dir2"));
    Path configStoreRoot = new Path(configRoot, SimpleHDFSConfigStore.CONFIG_STORE_NAME);
    Assert.assertTrue(localFS.mkdirs(configStoreRoot));
    try {/*from w  w w  .  j a  v  a 2 s.c  om*/
        Config confConf1 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(configRoot.toString()));
        SimpleHDFSConfigStoreFactory confFactory = new SimpleHDFSConfigStoreFactory(confConf1);
        Assert.assertTrue(confFactory.hasDefaultStoreURI());
        Assert.assertEquals(confFactory.getDefaultStoreURI(), configRoot.toUri());
        Assert.assertEquals(confFactory.getPhysicalScheme(), "file");
        Assert.assertEquals(confFactory.getDefaultRootDir().toString(),
                "file:" + System.getProperty("user.home"));

        // Valid path
        SimpleHDFSConfigStore store1 = confFactory.createConfigStore(new URI("simple-file:/d"));
        Assert.assertEquals(store1.getStoreURI().getScheme(), confFactory.getScheme());
        Assert.assertEquals(store1.getStoreURI().getAuthority(),
                confFactory.getDefaultStoreURI().getAuthority());
        Assert.assertEquals(store1.getStoreURI().getPath(), confFactory.getDefaultStoreURI().getPath());

        // Invalid path
        Config confConf2 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(testRoot.toString()));
        try {
            new SimpleHDFSConfigStoreFactory(confConf2);
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("Path does not appear to be a config store root"));
        }

        // Empty path
        Config confConf3 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(""));
        try {
            new SimpleHDFSConfigStoreFactory(confConf3);
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("Default store URI should be non-empty"));
        }
    } finally {
        localFS.delete(testRoot, true);
    }
}

From source file:gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter.java

License:Apache License

private void ensureDirectoryExists(FileSystem fs, Path path,
        Iterator<OwnerAndPermission> ownerAndPermissionIterator) throws IOException {

    if (fs.exists(path)) {
        return;/*w  w w .j a  v a  2s. c o m*/
    }

    if (ownerAndPermissionIterator.hasNext()) {
        OwnerAndPermission ownerAndPermission = ownerAndPermissionIterator.next();

        if (path.getParent() != null) {
            ensureDirectoryExists(fs, path.getParent(), ownerAndPermissionIterator);
        }

        if (!fs.mkdirs(path)) {
            // fs.mkdirs returns false if path already existed. Do not overwrite permissions
            return;
        }

        if (ownerAndPermission.getFsPermission() != null) {
            log.debug("Applying permissions %s to path %s.", ownerAndPermission.getFsPermission(), path);
            fs.setPermission(path, addExecutePermissionToOwner(ownerAndPermission.getFsPermission()));
        }

        String group = ownerAndPermission.getGroup();
        String owner = ownerAndPermission.getOwner();
        if (group != null || owner != null) {
            log.debug("Applying owner %s and group %s to path %s.", owner, group, path);
            fs.setOwner(path, owner, group);
        }
    } else {
        fs.mkdirs(path);
    }
}

From source file:gobblin.data.management.trash.TrashFactoryTest.java

License:Apache License

@Test
public void test() throws IOException {
    FileSystem fs = mock(FileSystem.class);

    Path homeDirectory = new Path("/home/directory");
    Path trashDirectory = new Path(homeDirectory, Trash.DEFAULT_TRASH_DIRECTORY);
    Path trashIdentifierFile = new Path(trashDirectory, Trash.TRASH_IDENTIFIER_FILE);

    when(fs.getHomeDirectory()).thenReturn(homeDirectory);
    when(fs.exists(trashDirectory)).thenReturn(true);
    when(fs.exists(trashIdentifierFile)).thenReturn(true);
    when(fs.listStatus(trashDirectory)).thenReturn(new FileStatus[] {});
    when(fs.isDirectory(trashDirectory)).thenReturn(true);

    when(fs.mkdirs(any(Path.class))).thenReturn(true);
    when(fs.mkdirs(any(Path.class), any(FsPermission.class))).thenReturn(true);
    when(fs.createNewFile(any(Path.class))).thenReturn(true);
    when(fs.makeQualified(any(Path.class))).thenAnswer(new Answer<Path>() {
        @Override/*ww  w .j  a  va2  s  .  c  o  m*/
        public Path answer(InvocationOnMock invocation) throws Throwable {
            return (Path) invocation.getArguments()[0];
        }
    });

    Properties properties;

    properties = getBaseProperties(trashDirectory);
    Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof Trash);
    Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof ProxiedTrash);

    properties = getBaseProperties(trashDirectory);
    properties.setProperty(TrashFactory.SIMULATE, Boolean.toString(true));
    Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof MockTrash);
    Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof MockTrash);

    properties = getBaseProperties(trashDirectory);
    properties.setProperty(TrashFactory.TRASH_TEST, Boolean.toString(true));
    Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof TestTrash);
    Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof TestTrash);

    properties = getBaseProperties(trashDirectory);
    properties.setProperty(TrashFactory.SKIP_TRASH, Boolean.toString(true));
    Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof ImmediateDeletionTrash);
    Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof ImmediateDeletionTrash);

}

From source file:gobblin.filesystem.InstrumentedHDFSFileSystemTest.java

License:Open Source License

/**
 * This test is disabled because it requires a local hdfs cluster at localhost:8020, which requires installation and setup.
 * Changes to {@link InstrumentedHDFSFileSystem} should be followed by a manual run of this tests.
 *
 * TODO: figure out how to fully automate this test.
 * @throws Exception/*from w w  w.ja  v  a  2s  .  c  om*/
 */
@Test(enabled = false)
public void test() throws Exception {

    FileSystem fs = FileSystem.get(new URI("instrumented-hdfs://localhost:8020"), new Configuration());

    String name = UUID.randomUUID().toString();
    fs.mkdirs(new Path("/tmp"));

    // Test absolute paths
    Path absolutePath = new Path("/tmp", name);
    Assert.assertFalse(fs.exists(absolutePath));
    fs.createNewFile(absolutePath);
    Assert.assertTrue(fs.exists(absolutePath));
    Assert.assertEquals(fs.getFileStatus(absolutePath).getLen(), 0);
    fs.delete(absolutePath, false);
    Assert.assertFalse(fs.exists(absolutePath));

    // Test fully qualified paths
    Path fqPath = new Path("instrumented-hdfs://localhost:8020/tmp", name);
    Assert.assertFalse(fs.exists(fqPath));
    fs.createNewFile(fqPath);
    Assert.assertTrue(fs.exists(fqPath));
    Assert.assertEquals(fs.getFileStatus(fqPath).getLen(), 0);
    fs.delete(fqPath, false);
    Assert.assertFalse(fs.exists(fqPath));
}

From source file:gobblin.filesystem.MetricsFileSystemInstrumentationTest.java

License:Apache License

/**
 * This test is disabled because it requires a local hdfs cluster at localhost:8020, which requires installation and setup.
 * Changes to {@link MetricsFileSystemInstrumentation} should be followed by a manual run of this tests.
 *
 * TODO: figure out how to fully automate this test.
 * @throws Exception//from  ww  w .j  a va  2 s  .co  m
 */
@Test(enabled = false)
public void test() throws Exception {

    String uri = "instrumented-hdfs://localhost:9000";

    FileSystem fs = FileSystem.get(new URI(uri), new Configuration());

    String name = UUID.randomUUID().toString();
    fs.mkdirs(new Path("/tmp"));

    // Test absolute paths
    Path absolutePath = new Path("/tmp", name);
    Assert.assertFalse(fs.exists(absolutePath));
    fs.createNewFile(absolutePath);
    Assert.assertTrue(fs.exists(absolutePath));
    Assert.assertEquals(fs.getFileStatus(absolutePath).getLen(), 0);
    fs.delete(absolutePath, false);
    Assert.assertFalse(fs.exists(absolutePath));

    // Test fully qualified paths
    Path fqPath = new Path(uri + "/tmp", name);
    Assert.assertFalse(fs.exists(fqPath));
    fs.createNewFile(fqPath);
    Assert.assertTrue(fs.exists(fqPath));
    Assert.assertEquals(fs.getFileStatus(fqPath).getLen(), 0);
    fs.delete(fqPath, false);
    Assert.assertFalse(fs.exists(fqPath));
}

From source file:gobblin.metrics.GobblinMetrics.java

License:Apache License

private void buildFileMetricReporter(Properties properties) {
    if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY,
            ConfigurationKeys.DEFAULT_METRICS_REPORTING_FILE_ENABLED))) {
        return;//  ww  w .j ava2s. c  o  m
    }
    LOGGER.info("Reporting metrics to log files");

    if (!properties.containsKey(ConfigurationKeys.METRICS_LOG_DIR_KEY)) {
        LOGGER.error("Not reporting metrics to log files because " + ConfigurationKeys.METRICS_LOG_DIR_KEY
                + " is undefined");
        return;
    }

    try {
        String fsUri = properties.getProperty(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI);
        FileSystem fs = FileSystem.get(URI.create(fsUri), new Configuration());

        // Each job gets its own metric log subdirectory
        Path metricsLogDir = new Path(properties.getProperty(ConfigurationKeys.METRICS_LOG_DIR_KEY),
                this.getName());
        if (!fs.exists(metricsLogDir) && !fs.mkdirs(metricsLogDir)) {
            LOGGER.error("Failed to create metric log directory for metrics " + this.getName());
            return;
        }

        // Add a suffix to file name if specified in properties.
        String metricsFileSuffix = properties.getProperty(ConfigurationKeys.METRICS_FILE_SUFFIX,
                ConfigurationKeys.DEFAULT_METRICS_FILE_SUFFIX);
        if (!Strings.isNullOrEmpty(metricsFileSuffix) && !metricsFileSuffix.startsWith(".")) {
            metricsFileSuffix = "." + metricsFileSuffix;
        }

        // Each job run gets its own metric log file
        Path metricLogFile = new Path(metricsLogDir, this.id + metricsFileSuffix + ".metrics.log");
        boolean append = false;
        // Append to the metric file if it already exists
        if (fs.exists(metricLogFile)) {
            LOGGER.info(String.format("Metric log file %s already exists, appending to it", metricLogFile));
            append = true;
        }

        OutputStream output = append ? fs.append(metricLogFile) : fs.create(metricLogFile, true);
        OutputStreamReporter.Factory.newBuilder().outputTo(output).build(properties);
        this.codahaleScheduledReporters.add(this.codahaleReportersCloser.register(
                OutputStreamEventReporter.forContext(RootMetricContext.get()).outputTo(output).build()));

        LOGGER.info("Will start reporting metrics to directory " + metricsLogDir);
    } catch (IOException ioe) {
        LOGGER.error("Failed to build file metric reporter for job " + this.id, ioe);
    }
}

From source file:gobblin.metrics.JobMetrics.java

License:Open Source License

private void buildFileMetricReporter(Properties properties) {
    if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY,
            ConfigurationKeys.DEFAULT_METRICS_REPORTING_FILE_ENABLED))) {
        LOGGER.info("Not reporting metrics to log files");
        return;//  w  ww  .  jav a2  s . co m
    }

    if (!properties.containsKey(ConfigurationKeys.METRICS_LOG_DIR_KEY)) {
        LOGGER.error("Not reporting metrics to log files because " + ConfigurationKeys.METRICS_LOG_DIR_KEY
                + " is undefined");
        return;
    }

    try {
        String fsUri = properties.getProperty(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI);
        FileSystem fs = FileSystem.get(URI.create(fsUri), new Configuration());

        // Each job gets its own metric log subdirectory
        Path metricsLogDir = new Path(properties.getProperty(ConfigurationKeys.METRICS_LOG_DIR_KEY),
                this.jobName);
        if (!fs.exists(metricsLogDir) && !fs.mkdirs(metricsLogDir)) {
            LOGGER.error("Failed to create metric log directory for job " + this.jobName);
            return;
        }

        // Each job run gets its own metric log file
        Path metricLogFile = new Path(metricsLogDir, this.jobId + ".metrics.log");
        boolean append = false;
        // Append to the metric file if it already exists
        if (fs.exists(metricLogFile)) {
            LOGGER.info(String.format("Metric log file %s already exists, appending to it", metricLogFile));
            append = true;
        }

        PrintStream ps = append ? this.closer.register(new PrintStream(fs.append(metricLogFile)))
                : this.closer.register(new PrintStream(fs.create(metricLogFile)));
        this.fileReporter = Optional.of(ConsoleReporter.forRegistry(this.metricRegistry).outputTo(ps)
                .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build());
    } catch (IOException ioe) {
        LOGGER.error("Failed to build file metric reporter for job " + this.jobId, ioe);
    }
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListFilesRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir1");
    try {// www . j a va2  s  .c  o m
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        List<FileStatus> testFiles = FileListUtils.listFilesRecursively(localFs, baseDir);

        Assert.assertEquals(2, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }
        Assert.assertTrue(fileNames.contains(TEST_FILE_NAME1) && fileNames.contains(TEST_FILE_NAME2));
    } finally {
        localFs.delete(baseDir, true);
    }
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListPathsRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir2");
    try {//w  w  w.  j av a  2  s .  c o m
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        List<FileStatus> testFiles = FileListUtils.listPathsRecursively(localFs, baseDir, new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return true;
            }
        });
        Assert.assertEquals(4, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }

        Set<String> expectedFileNames = Sets.newHashSet();
        expectedFileNames.add(baseDir.getName());
        expectedFileNames.add(subDir.getName());
        expectedFileNames.add(TEST_FILE_NAME1);
        expectedFileNames.add(TEST_FILE_NAME2);

        Assert.assertEquals(fileNames, expectedFileNames);
    } finally {
        localFs.delete(baseDir, true);
    }
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListMostNestedPathRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir3");
    String emptyDir1 = "emptyDir1";
    String emptyDir2 = "emptyDir2";
    try {//w w w  . j  a va 2 s  . c o  m
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        localFs.mkdirs(new Path(baseDir, emptyDir1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        localFs.mkdirs(new Path(subDir, emptyDir2));

        List<FileStatus> testFiles = FileListUtils.listMostNestedPathRecursively(localFs, baseDir);

        Assert.assertEquals(4, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }

        Set<String> expectedFileNames = Sets.newHashSet();
        expectedFileNames.add(emptyDir1);
        expectedFileNames.add(emptyDir2);
        expectedFileNames.add(TEST_FILE_NAME1);
        expectedFileNames.add(TEST_FILE_NAME2);

        Assert.assertEquals(fileNames, expectedFileNames);
    } finally {
        localFs.delete(baseDir, true);
    }
}