Example usage for org.apache.hadoop.fs FileSystem mkdirs

List of usage examples for org.apache.hadoop.fs FileSystem mkdirs

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:com.inmobi.conduit.AbstractService.java

License:Apache License

protected boolean retriableMkDirs(FileSystem fs, Path p, String streamName) throws Exception {
    int count = 0;
    boolean result = false;
    Exception ex = null;//  w  w  w. java2s .com
    while (count < numOfRetries) {
        try {
            result = fs.mkdirs(p);
            ex = null;
            break;

        } catch (Exception e) {
            LOG.warn("Couldn't make directories for path " + p + " .Retrying ", e);
            ex = e;
            if (stopped)
                break;
        }
        count++;
        if (streamName != null) {
            ConduitMetrics.updateSWGuage(getServiceType(), RETRY_MKDIR, streamName, 1);
        } else {
            LOG.warn("Can not increment retriable mkdir gauge as stream name is null");
        }
        try {
            Thread.sleep(TIME_RETRY_IN_MILLIS);
        } catch (InterruptedException e) {
            LOG.warn(e);
        }
    }
    if (count == numOfRetries) {
        LOG.error("Max retries done for mkdirs " + p + " quitting");
    }
    if (ex == null)
        return result;
    else
        throw ex;
}

From source file:com.inmobi.conduit.Conduit.java

License:Apache License

private void copyInputFormatJarToClusterFS(Cluster cluster, String inputFormatSrcJar) throws IOException {
    FileSystem clusterFS = FileSystem.get(cluster.getHadoopConf());
    // create jars path inside /conduit/system/tmp path
    Path jarsPath = new Path(cluster.getTmpPath(), "jars");
    if (!clusterFS.exists(jarsPath)) {
        clusterFS.mkdirs(jarsPath);
    }/*  w ww .  j  av  a2 s . c  o m*/
    // copy inputFormat source jar into /conduit/system/tmp/jars path
    Path inputFormatJarDestPath = new Path(jarsPath, "conduit-distcp-current.jar");
    if (clusterFS.exists(inputFormatJarDestPath)) {
        clusterFS.delete(inputFormatJarDestPath, true);
    }
    clusterFS.copyFromLocalFile(new Path(inputFormatSrcJar), inputFormatJarDestPath);
}

From source file:com.inmobi.conduit.Conduit.java

License:Apache License

private void copyAuditUtilJarToClusterFs(Cluster cluster, String auditUtilSrcJar) throws IOException {
    FileSystem clusterFS = FileSystem.get(cluster.getHadoopConf());
    // create jars path inside /conduit/system/tmp path
    Path jarsPath = new Path(cluster.getTmpPath(), "jars");
    if (!clusterFS.exists(jarsPath)) {
        clusterFS.mkdirs(jarsPath);
    }//w w w . ja  v a  2s . c  o  m
    // copy AuditUtil source jar into /conduit/system/tmp/jars path
    Path AuditUtilJarDestPath = new Path(jarsPath, "messaging-client-core.jar");
    if (clusterFS.exists(AuditUtilJarDestPath)) {
        clusterFS.delete(AuditUtilJarDestPath, true);
    }
    clusterFS.copyFromLocalFile(new Path(auditUtilSrcJar), AuditUtilJarDestPath);
}

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableDirectoryCreateCommand.java

License:Apache License

/**
 * Implementation of RetriableCommand::doExecute().
 * This implements the actual mkdirs() functionality.
 * @param arguments Argument-list to the command.
 * @return Boolean. True, if the directory could be created successfully.
 * @throws Exception IOException, on failure to create the directory.
 *//*ww w . j  av  a  2 s.c o  m*/
@Override
protected Object doExecute(Object... arguments) throws Exception {
    assert arguments.length == 2 : "Unexpected argument list.";
    Path target = (Path) arguments[0];
    Mapper.Context context = (Mapper.Context) arguments[1];

    FileSystem targetFS = target.getFileSystem(HadoopCompat.getTaskConfiguration(context));
    return targetFS.mkdirs(target);
}

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void promoteTmpToTarget(Path tmpTarget, Path target, FileSystem fs) throws IOException {
    if ((fs.exists(target) && !fs.delete(target, false))
            || (!fs.exists(target.getParent()) && !fs.mkdirs(target.getParent()))
            || !fs.rename(tmpTarget, target)) {
        throw new IOException("Failed to promote tmp-file:" + tmpTarget + " to: " + target);
    }/*from  www.jav  a  2s . c o m*/
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitMissingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;
    try {/*  w  w w .  j  a  v a 2 s .c o  m*/
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertFalse(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitExistingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class, Mockito.RETURNS_DEEP_STUBS);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;
    try {//from  w  w w .  j  a v a  2 s  .c  om
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));
        fs.mkdirs(new Path(finalPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private static void mkdirs(String path) throws Exception {
    FileSystem fileSystem = cluster.getFileSystem();
    final Path qualifiedPath = new Path(path).makeQualified(fileSystem);
    pathList.add(qualifiedPath);/* www. ja  v a2  s  .  c  om*/
    fileSystem.mkdirs(qualifiedPath);
}

From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java

License:Apache License

@Test
public void testMultipleSrcToFile() {
    FileSystem fs = null;
    try {//ww w  .  j av  a 2 s.  c o m
        fs = FileSystem.get(getConf());
        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(new Path("/tmp/in/1"));
        srcPaths.add(new Path("/tmp/in/2"));
        Path target = new Path("/tmp/out/1");
        TestDistCpUtils.createFile(fs, "/tmp/in/1");
        TestDistCpUtils.createFile(fs, "/tmp/in/2");
        fs.mkdirs(target);
        DistCpOptions options = new DistCpOptions(srcPaths, target);
        validatePaths(options);
        TestDistCpUtils.delete(fs, "/tmp");
        //No errors

        target = new Path("/tmp/out/1");
        fs.create(target).close();
        options = new DistCpOptions(srcPaths, target);
        try {
            validatePaths(options);
            Assert.fail("Invalid inputs accepted");
        } catch (InvalidInputException ignore) {
        }
        TestDistCpUtils.delete(fs, "/tmp");

        srcPaths.clear();
        srcPaths.add(new Path("/tmp/in/1"));
        fs.mkdirs(new Path("/tmp/in/1"));
        target = new Path("/tmp/out/1");
        fs.create(target).close();
        options = new DistCpOptions(srcPaths, target);
        try {
            validatePaths(options);
            Assert.fail("Invalid inputs accepted");
        } catch (InvalidInputException ignore) {
        }
        TestDistCpUtils.delete(fs, "/tmp");
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test input validation failed");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp");
    }
}

From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java

License:Apache License

@Test
public void testBuildListing() {
    FileSystem fs = null;
    try {//from w w w.  j a  va  2s. c o  m
        fs = FileSystem.get(getConf());
        List<Path> srcPaths = new ArrayList<Path>();
        Path p1 = new Path("/tmp/in/1");
        Path p2 = new Path("/tmp/in/2");
        Path p3 = new Path("/tmp/in2/2");
        Path target = new Path("/tmp/out/1");
        srcPaths.add(p1.getParent());
        srcPaths.add(p3.getParent());
        TestDistCpUtils.createFile(fs, "/tmp/in/1");
        TestDistCpUtils.createFile(fs, "/tmp/in/2");
        TestDistCpUtils.createFile(fs, "/tmp/in2/2");
        fs.mkdirs(target);
        OutputStream out = fs.create(p1);
        out.write("ABC".getBytes());
        out.close();

        out = fs.create(p2);
        out.write("DEF".getBytes());
        out.close();

        out = fs.create(p3);
        out.write("GHIJ".getBytes());
        out.close();

        Path listingFile = new Path("/tmp/file");

        DistCpOptions options = new DistCpOptions(srcPaths, target);
        options.setSyncFolder(true);
        CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
        try {
            listing.buildListing(listingFile, options);
            Assert.fail("Duplicates not detected");
        } catch (DuplicateFileException ignore) {
        }
        Assert.assertEquals(listing.getBytesToCopy(), 10);
        Assert.assertEquals(listing.getNumberOfPaths(), 3);
        TestDistCpUtils.delete(fs, "/tmp");

        try {
            listing.buildListing(listingFile, options);
            Assert.fail("Invalid input not detected");
        } catch (InvalidInputException ignore) {
        }
        TestDistCpUtils.delete(fs, "/tmp");
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test build listing failed");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp");
    }
}