Example usage for org.apache.hadoop.fs FileStatus getPath

List of usage examples for org.apache.hadoop.fs FileStatus getPath

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath()

Source Link

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void compareFileLengths(FileStatus sourceFileStatus, Path target, Configuration configuration,
        long bytesRead) throws IOException {
    final Path sourcePath = sourceFileStatus.getPath();
    FileSystem fs = sourcePath.getFileSystem(configuration);
    if (fs.getFileStatus(sourcePath).getLen() != bytesRead)
        throw new IOException("Mismatch in length of source:" + sourcePath + " and target:" + target);
}

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, int bufferSize,
        Mapper.Context context, Map<Long, Long> received) throws IOException {
    Path source = sourceFileStatus.getPath();
    ThrottledInputStream inStream = null;
    final CompressionCodec codec = compressionCodecs.getCodec(source);
    InputStream compressedIn = null;
    OutputStream commpressedOut = null;
    BufferedReader reader = null;
    long numberOfLinesRead = 0;

    try {//ww w .j a v a  2 s  .c o m
        inStream = getInputStream(source, HadoopCompat.getTaskConfiguration(context));
        compressedIn = codec.createInputStream(inStream);
        commpressedOut = codec.createOutputStream(outStream);
        // LineReader reader = new LineReader(compressedIn,
        // context.getConfiguration(), null);
        reader = new BufferedReader(new InputStreamReader(compressedIn));
        byte[] bytesRead = readLine(reader);
        while (bytesRead != null) {
            numberOfLinesRead++;
            commpressedOut.write(bytesRead);
            commpressedOut.write("\n".getBytes());
            updateContextStatus(inStream.getTotalBytesRead(), context, sourceFileStatus, numberOfLinesRead);
            if (received != null) {
                byte[] decodedMsg = Base64.decodeBase64(bytesRead);
                incrementReceived(decodedMsg, received);
            }
            bytesRead = readLine(reader);
        }
        HadoopCompat.incrementCounter(HadoopCompat.getCounter(context, CopyMapper.Counter.SLEEP_TIME_MS),
                inStream.getTotalSleepTime());
        LOG.info("STATS: " + inStream);
    } finally {
        IOUtils.cleanup(LOG, inStream, reader, compressedIn);
        try {
            if (commpressedOut != null)
                commpressedOut.close();
            outStream.close();
        } catch (IOException exception) {
            LOG.error("Could not close output-stream. ", exception);
            throw exception;
        }
    }

    return inStream.getTotalBytesRead();// totalBytesRead;
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

private boolean checkDirectoryPermissions(FileSystem fs, String targetBase, FsPermission sourcePerm)
        throws IOException {
    Path base = new Path(targetBase);

    Stack<Path> stack = new Stack<Path>();
    stack.push(base);/*from   w  ww.ja  v  a 2  s . co  m*/
    while (!stack.isEmpty()) {
        Path file = stack.pop();
        if (!fs.exists(file))
            continue;
        FileStatus[] fStatus = fs.listStatus(file);
        if (fStatus == null || fStatus.length == 0)
            continue;

        for (FileStatus status : fStatus) {
            if (status.isDir()) {
                stack.push(status.getPath());
                Assert.assertEquals(status.getPermission(), sourcePerm);
            }
        }
    }
    return true;
}

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

/** {@inheritDoc} */
@Override//from   w  w w.j  av  a  2  s. c om
public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException {

    SequenceFile.Writer fileListWriter = null;

    try {
        fileListWriter = getWriter(pathToListingFile);

        for (Path path : options.getSourcePaths()) {
            FileSystem sourceFS = path.getFileSystem(getConf());
            path = makeQualified(path);

            FileStatus rootStatus = sourceFS.getFileStatus(path);
            Path sourcePathRoot = computeSourceRootPath(rootStatus, options);
            boolean localFile = (rootStatus.getClass() != FileStatus.class);

            FileStatus[] sourceFiles = sourceFS.listStatus(path);
            if (sourceFiles != null && sourceFiles.length > 0) {
                for (FileStatus sourceStatus : sourceFiles) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
                    }
                    writeToFileListing(fileListWriter, sourceStatus, sourcePathRoot, localFile, options);

                    if (isDirectoryAndNotEmpty(sourceFS, sourceStatus)) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Traversing non-empty source dir: " + sourceStatus.getPath());
                        }
                        traverseNonEmptyDirectory(fileListWriter, sourceStatus, sourcePathRoot, localFile,
                                options);
                    }
                }
            } else {
                writeToFileListing(fileListWriter, rootStatus, sourcePathRoot, localFile, options);
            }
        }
    } finally {
        try {
            if (fileListWriter != null)
                fileListWriter.close();
        } catch (IOException exception) {
            LOG.error("Could not close output-steam to the file-list: ", exception);
            throw exception;
        }
    }
}

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private Path computeSourceRootPath(FileStatus sourceStatus, DistCpOptions options) throws IOException {

    Path target = options.getTargetPath();
    FileSystem targetFS = target.getFileSystem(getConf());

    boolean solitaryFile = options.getSourcePaths().size() == 1 && !sourceStatus.isDir();

    if (solitaryFile) {
        if (targetFS.isFile(target) || !targetFS.exists(target)) {
            return sourceStatus.getPath();
        } else {//from   w  w w.  ja  v  a  2  s.  c  o m
            return sourceStatus.getPath().getParent();
        }
    } else {
        boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetFS.exists(target))
                || options.shouldSyncFolder() || options.shouldOverwrite();

        return specialHandling && sourceStatus.isDir() ? sourceStatus.getPath()
                : sourceStatus.getPath().getParent();
    }
}

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private static FileStatus[] getChildren(FileSystem fileSystem, FileStatus parent) throws IOException {
    return fileSystem.listStatus(parent.getPath());
}

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter, FileStatus sourceStatus,
        Path sourcePathRoot, boolean localFile, DistCpOptions options) throws IOException {
    FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf());
    Stack<FileStatus> pathStack = new Stack<FileStatus>();
    pathStack.push(sourceStatus);/* ww w. jav  a 2s .  c o  m*/

    while (!pathStack.isEmpty()) {
        for (FileStatus child : getChildren(sourceFS, pathStack.pop())) {
            if (LOG.isDebugEnabled())
                LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
            writeToFileListing(fileListWriter, child, sourcePathRoot, localFile, options);
            if (isDirectoryAndNotEmpty(sourceFS, child)) {
                if (LOG.isDebugEnabled())
                    LOG.debug("Traversing non-empty source dir: " + sourceStatus.getPath());
                pathStack.push(child);
            }
        }
    }
}

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private void writeToFileListing(SequenceFile.Writer fileListWriter, FileStatus fileStatus, Path sourcePathRoot,
        boolean localFile, DistCpOptions options) throws IOException {
    if (fileStatus.getPath().equals(sourcePathRoot) && fileStatus.isDir())
        return; // Skip the root-paths.

    if (LOG.isDebugEnabled()) {
        LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath())
                + ", FULL PATH: " + fileStatus.getPath());
    }//from  ww  w  .  j  a va2s. c o m

    FileStatus status = fileStatus;
    if (localFile) {
        status = getFileStatus(fileStatus);
    }

    if (!shouldCopy(fileStatus.getPath(), options))
        return;
    if (options.shouldPreserveSrcPath()) {
        fileListWriter.append(new Text(fileStatus.getPath().toUri().getPath()), status);
    } else {
        fileListWriter.append(new Text(DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath())),
                status);
    }

    fileListWriter.sync();

    if (!fileStatus.isDir()) {
        totalBytesToCopy += fileStatus.getLen();
    }
    totalPaths++;
}

From source file:com.inmobi.conduit.distcp.tools.TestFileBasedCopyListing.java

License:Apache License

private void checkResult(Path listFile, int count) throws IOException {
    if (count == 0) {
        return;// w w  w.j a  v  a2s .c  o  m
    }

    int recCount = 0;
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, listFile, config);
    try {
        Text relPath = new Text();
        FileStatus fileStatus = new FileStatus();
        while (reader.next(relPath, fileStatus)) {
            Assert.assertEquals(fileStatus.getPath().toUri().getPath(), map.get(relPath.toString()));
            recCount++;
        }
    } finally {
        IOUtils.closeStream(reader);
    }
    Assert.assertEquals(recCount, count);
}

From source file:com.inmobi.conduit.distcp.tools.TestGlobbedCopyListing.java

License:Apache License

private void verifyContents(Path listingPath) throws Exception {
    SequenceFile.Reader reader = new SequenceFile.Reader(cluster.getFileSystem(), listingPath,
            new Configuration());
    Text key = new Text();
    FileStatus value = new FileStatus();
    Map<String, String> actualValues = new HashMap<String, String>();
    while (reader.next(key, value)) {
        actualValues.put(value.getPath().toString(), key.toString());
    }//ww w .  jav a2 s. c o m

    Assert.assertEquals(expectedValues.size(), actualValues.size());
    for (Map.Entry<String, String> entry : actualValues.entrySet()) {
        Assert.assertEquals(entry.getValue(), expectedValues.get(entry.getKey()));
    }
}