Example usage for org.apache.hadoop.fs FileSystem exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException 

Source Link

Document

Check if a path exists.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private long doCopy(FileStatus sourceFileStatus, Path target, Mapper.Context context,
        EnumSet<FileAttribute> fileAttributes, Map<Long, Long> received) throws IOException {

    Path tmpTargetPath = getTmpFile(target, context);
    final Configuration configuration = HadoopCompat.getTaskConfiguration(context);
    FileSystem targetFS = target.getFileSystem(configuration);
    compressionCodecs = new CompressionCodecFactory(context.getConfiguration());
    try {/*  w  w w. ja va 2s  . co m*/
        if (LOG.isDebugEnabled()) {
            LOG.debug("Copying " + sourceFileStatus.getPath() + " to " + target);
            LOG.debug("Tmp-file path: " + tmpTargetPath);
        }
        FileSystem sourceFS = sourceFileStatus.getPath().getFileSystem(configuration);
        long bytesRead = copyToTmpFile(tmpTargetPath, targetFS, sourceFileStatus, context, fileAttributes,
                received);

        compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead);
        if (bytesRead > 0) {
            compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath);
        }
        promoteTmpToTarget(tmpTargetPath, target, targetFS);
        return bytesRead;

    } finally {
        if (targetFS.exists(tmpTargetPath))
            targetFS.delete(tmpTargetPath, false);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void promoteTmpToTarget(Path tmpTarget, Path target, FileSystem fs) throws IOException {
    if ((fs.exists(target) && !fs.delete(target, false))
            || (!fs.exists(target.getParent()) && !fs.mkdirs(target.getParent()))
            || !fs.rename(tmpTarget, target)) {
        throw new IOException("Failed to promote tmp-file:" + tmpTarget + " to: " + target);
    }//from   w ww  . java  2  s  . c om
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitMissingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;
    try {//w ww .j  a  va 2  s .c om
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertFalse(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testAtomicCommitExistingFinal() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class, Mockito.RETURNS_DEEP_STUBS);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String workPath = "/tmp1/" + String.valueOf(rand.nextLong());
    String finalPath = "/tmp1/" + String.valueOf(rand.nextLong());
    FileSystem fs = null;
    try {//from  w  w w  . j a  va2 s  . c om
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        fs.mkdirs(new Path(workPath));
        fs.mkdirs(new Path(finalPath));

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true);
        //XXX set label to false explicitly, conf is not mixed up
        conf.setBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false);

        Assert.assertTrue(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

        //Test for idempotent commit
        committer.commitJob(jobContext);
        Assert.assertFalse(fs.exists(new Path(workPath)));
        Assert.assertTrue(fs.exists(new Path(finalPath)));

    } catch (IOException e) {
        LOG.error("Exception encountered while testing for preserve status", e);
        Assert.fail("Atomic commit failure");
    } finally {
        TestDistCpUtils.delete(fs, workPath);
        TestDistCpUtils.delete(fs, finalPath);
        conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

private boolean checkDirectoryPermissions(FileSystem fs, String targetBase, FsPermission sourcePerm)
        throws IOException {
    Path base = new Path(targetBase);

    Stack<Path> stack = new Stack<Path>();
    stack.push(base);//from ww w  .  j a va 2  s. c o m
    while (!stack.isEmpty()) {
        Path file = stack.pop();
        if (!fs.exists(file))
            continue;
        FileStatus[] fStatus = fs.listStatus(file);
        if (fStatus == null || fStatus.length == 0)
            continue;

        for (FileStatus status : fStatus) {
            if (status.isDir()) {
                stack.push(status.getPath());
                Assert.assertEquals(status.getPermission(), sourcePerm);
            }
        }
    }
    return true;
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testRun() {
    try {// w ww  .j a  va2s  .c  om
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);
        copyMapper.setup(context);

        for (Path path : pathList) {
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                    fs.getFileStatus(path), context);
        }
        // Check that the maps worked.
        for (Path path : pathList) {
            final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
            Assert.assertTrue(fs.exists(targetPath));
            Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path));
            Assert.assertEquals(fs.getFileStatus(path).getReplication(),
                    fs.getFileStatus(targetPath).getReplication());
            Assert.assertEquals(fs.getFileStatus(path).getBlockSize(),
                    fs.getFileStatus(targetPath).getBlockSize());
            Assert.assertTrue(
                    !fs.isFile(targetPath) || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path)));
        }

        Assert.assertEquals(pathList.size(), reporter.getCounter(CopyMapper.Counter.PATHS_COPIED).getValue());
        // Here file is compressed file. So, we should compare the file length
        // with the number of bytes read
        long totalSize = 0;
        for (Path path : pathList) {
            totalSize += fs.getFileStatus(path).getLen();
        }
        Assert.assertEquals(totalSize, reporter.getCounter(CopyMapper.Counter.BYTES_COPIED).getValue());
        long totalCounterValue = 0;
        for (Text value : writer.values()) {
            String tmp[] = value.toString().split(ConduitConstants.AUDIT_COUNTER_NAME_DELIMITER);
            Assert.assertEquals(4, tmp.length);
            Long numOfMsgs = Long.parseLong(tmp[3]);
            totalCounterValue += numOfMsgs;
        }
        Assert.assertEquals(nFiles * NUMBER_OF_MESSAGES_PER_FILE, totalCounterValue);
        testCopyingExistingFiles(fs, copyMapper, context);
    } catch (Exception e) {
        LOG.error("Unexpected exception: ", e);
        Assert.assertTrue(false);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.UniformSizeInputFormat.java

License:Apache License

private SequenceFile.Reader getListingFileReader(Configuration configuration) {

    final Path listingFilePath = getListingFilePath(configuration);
    try {/*w  w w.j a va  2s  .c  o m*/
        final FileSystem fileSystem = listingFilePath.getFileSystem(configuration);
        if (!fileSystem.exists(listingFilePath))
            throw new IllegalArgumentException("Listing file doesn't exist at: " + listingFilePath);

        return new SequenceFile.Reader(fileSystem, listingFilePath, configuration);
    } catch (IOException exception) {
        LOG.error("Couldn't find listing file at: " + listingFilePath, exception);
        throw new IllegalArgumentException("Couldn't find listing-file at: " + listingFilePath, exception);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

@Override
protected void validatePaths(DistCpOptions options) throws IOException, InvalidInputException {

    if (options.isSkipPathValidation()) {
        LOG.debug("Skipping Path Validation in disctp");
        return;//from   w  w  w  .jav  a  2  s.c  o m
    }

    Path targetPath = options.getTargetPath();
    FileSystem targetFS = targetPath.getFileSystem(getConf());
    boolean targetIsFile = targetFS.isFile(targetPath);

    //If target is a file, then source has to be single file
    if (targetIsFile) {
        if (options.getSourcePaths().size() > 1) {
            throw new InvalidInputException("Multiple source being copied to a file: " + targetPath);
        }

        Path srcPath = options.getSourcePaths().get(0);
        FileSystem sourceFS = srcPath.getFileSystem(getConf());
        if (!sourceFS.isFile(srcPath)) {
            throw new InvalidInputException(
                    "Cannot copy " + srcPath + ", which is not a file to " + targetPath);
        }
    }

    for (Path path : options.getSourcePaths()) {
        FileSystem fs = path.getFileSystem(getConf());
        if (!fs.exists(path)) {
            throw new InvalidInputException(path + " doesn't exist");
        }
    }

    /* This is requires to allow map tasks to access each of the source
       clusters. This would retrieve the delegation token for each unique
       file system and add them to job's private credential store
     */
    Credentials credentials = getCredentials();
    if (credentials != null) {
        Path[] inputPaths = options.getSourcePaths().toArray(new Path[1]);
        TokenCache.obtainTokensForNamenodes(credentials, inputPaths, getConf());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private Path computeSourceRootPath(FileStatus sourceStatus, DistCpOptions options) throws IOException {

    Path target = options.getTargetPath();
    FileSystem targetFS = target.getFileSystem(getConf());

    boolean solitaryFile = options.getSourcePaths().size() == 1 && !sourceStatus.isDir();

    if (solitaryFile) {
        if (targetFS.isFile(target) || !targetFS.exists(target)) {
            return sourceStatus.getPath();
        } else {/*from w ww  .j  av  a2 s .c om*/
            return sourceStatus.getPath().getParent();
        }
    } else {
        boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetFS.exists(target))
                || options.shouldSyncFolder() || options.shouldOverwrite();

        return specialHandling && sourceStatus.isDir() ? sourceStatus.getPath()
                : sourceStatus.getPath().getParent();
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java

License:Apache License

@Test
public void testBuildListingForSingleFile() {
    FileSystem fs = null;
    String testRootString = "/singleFileListing";
    Path testRoot = new Path(testRootString);
    SequenceFile.Reader reader = null;
    try {/*w  w  w .  j  ava 2 s .  co  m*/
        fs = FileSystem.get(getConf());
        if (fs.exists(testRoot))
            TestDistCpUtils.delete(fs, testRootString);

        Path sourceFile = new Path(testRoot, "/source/foo/bar/source.txt");
        Path decoyFile = new Path(testRoot, "/target/moo/source.txt");
        Path targetFile = new Path(testRoot, "/target/moo/target.txt");

        TestDistCpUtils.createFile(fs, sourceFile.toString());
        TestDistCpUtils.createFile(fs, decoyFile.toString());
        TestDistCpUtils.createFile(fs, targetFile.toString());

        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(sourceFile);

        DistCpOptions options = new DistCpOptions(srcPaths, targetFile);
        CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);

        final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
        listing.buildListing(listFile, options);

        reader = new SequenceFile.Reader(fs, listFile, getConf());
        FileStatus fileStatus = new FileStatus();
        Text relativePath = new Text();
        Assert.assertTrue(reader.next(relativePath, fileStatus));
        Assert.assertTrue(relativePath.toString().equals(""));
    } catch (Exception e) {
        Assert.fail("Unexpected exception encountered.");
        LOG.error("Unexpected exception: ", e);
    } finally {
        TestDistCpUtils.delete(fs, testRootString);
        IOUtils.closeStream(reader);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »