Example usage for org.apache.hadoop.fs FileStatus getPath

List of usage examples for org.apache.hadoop.fs FileStatus getPath

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath()

Source Link

Usage

From source file:com.asakusafw.operation.tools.hadoop.fs.Clean.java

License:Apache License

private boolean remove(FileSystem fs, FileStatus file, Context context) {
    LOG.debug("Attempt to remove {}", file.getPath()); //$NON-NLS-1$
    boolean isSymlink = context.isSymlink(fs, file);
    if (isSymlink) {
        LOG.error(MessageFormat.format("[OT-CLEAN-W01001] Symlink is currenty not supported: {0}",
                file.getPath()));//w w w.  ja va 2s . c  o  m
        context.setError();
        return false;
    }
    if (file.isDir()) {
        if (context.isRecursive()) {
            List<FileStatus> children;
            try {
                children = asList(fs.listStatus(file.getPath()));
            } catch (IOException e) {
                LOG.error(
                        MessageFormat.format("[OT-CLEAN-E01003] Failed to list directory: {0}", file.getPath()),
                        e);
                context.setError();
                return false;
            }
            boolean deleteChildren = true;
            for (FileStatus child : children) {
                deleteChildren &= remove(fs, child, context);
            }
            if (deleteChildren == false) {
                LOG.info(MessageFormat.format("[OT-CLEAN-I01004] Skipped: {0} (is no-empty directory)",
                        file.getPath(), new Date(file.getModificationTime())));
                return false;
            }
        } else {
            LOG.info(MessageFormat.format("[OT-CLEAN-I01003] Skipped: {0} (is directory)", file.getPath(),
                    new Date(file.getModificationTime())));
            return false;
        }
    }
    if (context.canDelete(file)) {
        LOG.debug("Removing {}", file.getPath()); //$NON-NLS-1$
        if (context.isDryRun() == false) {
            try {
                boolean removed = fs.delete(file.getPath(), false);
                if (removed == false) {
                    LOG.error(MessageFormat.format("[OT-CLEAN-E01004] Failed to remove: {0}", file.getPath()));
                    context.setError();
                    return false;
                }
            } catch (IOException e) {
                LOG.warn(MessageFormat.format("[OT-CLEAN-E01004] Failed to remove: {0}", file.getPath()), e);
                context.setError();
                return false;
            }
        }
        LOG.info(MessageFormat.format("[OT-CLEAN-I01001] Removed: {0} (timestamp={1})", file.getPath(),
                new Date(file.getModificationTime())));
    } else {
        LOG.info(MessageFormat.format("[OT-CLEAN-I01002] Kept: {0} (timestamp={1})", file.getPath(),
                new Date(file.getModificationTime())));
        return false;
    }
    return true;
}

From source file:com.asakusafw.runtime.compatibility.hadoop1.SequenceFileCompatibilityHadoop1.java

License:Apache License

@Override
public SequenceFile.Reader openReader(InputStream in, long length, Configuration conf) throws IOException {
    if (in == null) {
        throw new IllegalArgumentException("in must not be null"); //$NON-NLS-1$
    }// w  w  w .  ja  v  a2 s.  co  m
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    FileStatus status = new FileStatus(length, false, 0, length, 0, new Path("dummy:///")); //$NON-NLS-1$
    Path path = status.getPath();
    return new SequenceFile.Reader(new InputStreamFileSystem(status, in), path, conf);
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private TransactionInfo toInfoObject(FileStatus stat) throws IOException {
    assert stat != null;
    Path path = stat.getPath();
    String executionId = HadoopDataSourceUtil.getTransactionInfoExecutionId(path);
    long timestamp = stat.getModificationTime();
    List<String> comment = new ArrayList<>();
    Path commitMarkPath = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = path.getFileSystem(getConf());
    boolean committed = fs.exists(commitMarkPath);
    try (FSDataInputStream input = fs.open(path);
            Scanner scanner = new Scanner(new InputStreamReader(input, HadoopDataSourceUtil.COMMENT_CHARSET))) {
        while (scanner.hasNextLine()) {
            comment.add(scanner.nextLine());
        }// w  w  w .j a v  a 2 s .c om
    } catch (IOException e) {
        comment.add(e.toString());
    }

    return new TransactionInfo(executionId, timestamp, committed, comment);
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public <T> List<DirectInputFragment> findInputFragments(DataDefinition<T> definition, String basePath,
        ResourcePattern resourcePattern) throws IOException, InterruptedException {
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start finding input (id={0}, path={1}, resourcePattern={2})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern));
    }//ww  w.j  a  v  a 2 s.c o m
    FilePattern pattern = validate(resourcePattern);
    HadoopDataSourceProfile p = profile;
    FileSystem fs = p.getFileSystem();
    Path root = p.getFileSystemPath();
    Path base = append(root, basePath);
    Path temporary = p.getTemporaryFileSystemPath();
    List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);
    stats = filesOnly(stats, temporary);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process finding input (id={0}, path={1}, resource={2}, files={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, stats.size()));
    }
    if (LOG.isTraceEnabled()) {
        for (FileStatus stat : stats) {
            LOG.trace(MessageFormat.format("Input found (path={0}, length={1})", //$NON-NLS-1$
                    stat.getPath(), stat.getLen()));
        }
    }
    DataFilter<?> filter = definition.getDataFilter();
    if (filter != null) {
        stats = applyFilter(stats, filter);
    }

    DataFormat<T> format = definition.getDataFormat();
    Class<? extends T> dataType = definition.getDataClass();
    List<DirectInputFragment> results;
    if (format instanceof StripedDataFormat<?>) {
        StripedDataFormat.InputContext context = new StripedDataFormat.InputContext(dataType, stats, fs,
                p.getMinimumFragmentSize(), p.getPreferredFragmentSize(), p.isSplitBlocks(),
                p.isCombineBlocks());
        StripedDataFormat<T> sformat = (StripedDataFormat<T>) format;
        results = sformat.computeInputFragments(context);
    } else if (format instanceof FragmentableDataFormat<?>) {
        FragmentableDataFormat<T> sformat = (FragmentableDataFormat<T>) format;
        FragmentComputer optimizer = new FragmentComputer(p.getMinimumFragmentSize(sformat),
                p.getPreferredFragmentSize(sformat), p.isCombineBlocks(), p.isSplitBlocks());
        results = computeInputFragments(optimizer, stats);
    } else {
        FragmentComputer optimizer = new FragmentComputer();
        results = computeInputFragments(optimizer, stats);
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish finding input (id={0}, path={1}, resource={2}, fragments={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, results.size()));
    }
    return results;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

private List<FileStatus> applyFilter(List<FileStatus> stats, DataFilter<?> filter) {
    List<FileStatus> results = new ArrayList<>();
    for (FileStatus stat : stats) {
        String path = stat.getPath().toString();
        if (filter.acceptsPath(path)) {
            results.add(stat);/*w  w  w  .j  a v  a  2s.c  o  m*/
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("filtered direct input file: {0} ({1})", path, filter));
            }
        }
    }
    return results;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

private boolean isIn(FileStatus stat, Path temporary) {
    assert stat != null;
    assert temporary != null;
    Path path = stat.getPath();
    if (path.equals(temporary) || HadoopDataSourceUtil.contains(temporary, path)) {
        return true;
    }//from www.ja  va 2s .  c om
    return false;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

private List<DirectInputFragment> computeInputFragments(FragmentComputer fragmentComputer,
        List<FileStatus> stats) throws IOException {
    List<DirectInputFragment> results = new ArrayList<>();
    for (FileStatus stat : stats) {
        String path = stat.getPath().toString();
        long fileSize = stat.getLen();
        List<BlockInfo> blocks = BlockMap.computeBlocks(profile.getFileSystem(), stat);
        if (LOG.isTraceEnabled()) {
            for (BlockInfo block : blocks) {
                LOG.trace(MessageFormat.format("Original BlockInfo (path={0}, start={1}, end={2}, hosts={3})", //$NON-NLS-1$
                        path, block.getStart(), block.getEnd(), block.getHosts()));
            }/*from   w  w  w.  j a v a  2s. com*/
        }
        List<DirectInputFragment> fragments = fragmentComputer.computeFragments(path, fileSize, blocks);
        if (LOG.isTraceEnabled()) {
            for (DirectInputFragment fragment : fragments) {
                LOG.trace(MessageFormat.format("Fragment found (path={0}, offset={1}, size={2}, owners={3})", //$NON-NLS-1$
                        fragment.getPath(), fragment.getOffset(), fragment.getSize(),
                        fragment.getOwnerNodeNames()));
            }
        }
        results.addAll(fragments);
    }
    return results;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public List<ResourceInfo> list(String basePath, ResourcePattern resourcePattern, Counter counter)
        throws IOException, InterruptedException {
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start listing files (id={0}, path={1}, resource={2})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern));
    }/*  w  w  w .j a v  a2 s. c o m*/
    FilePattern pattern = validate(resourcePattern);
    HadoopDataSourceProfile p = profile;
    FileSystem fs = p.getFileSystem();
    Path root = p.getFileSystemPath();
    Path base = append(root, basePath);
    Path temporary = p.getTemporaryFileSystemPath();
    List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);
    stats = normalize(stats, root, temporary);

    List<ResourceInfo> results = new ArrayList<>();
    for (FileStatus stat : stats) {
        counter.add(1);
        ResourceInfo resource = new ResourceInfo(profile.getId(), stat.getPath().toString(),
                FileSystemCompatibility.isDirectory(stat));
        results.add(resource);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish listing files (id={0}, path={1}, resource={2}, count={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, results.size()));
    }
    return results;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public boolean delete(String basePath, ResourcePattern resourcePattern, boolean recursive, Counter counter)
        throws IOException, InterruptedException {
    assert basePath.startsWith("/") == false; //$NON-NLS-1$
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start deleting files (id={0}, path={1}, resource={2}, recursive={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, recursive));
    }//ww  w . j av  a 2  s.c  om
    FilePattern pattern = validate(resourcePattern);
    HadoopDataSourceProfile p = profile;
    FileSystem fs = p.getFileSystem();
    Path root = p.getFileSystemPath();
    Path base = append(root, basePath);
    List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);
    Path temporary = p.getTemporaryFileSystemPath();
    stats = normalize(stats, root, temporary);
    if (recursive) {
        stats = HadoopDataSourceUtil.onlyMinimalCovered(stats);
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process deleting files (id={0}, path={1}, resource={2}, files={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, stats.size()));
    }
    boolean succeed = true;
    for (FileStatus stat : stats) {
        if (LOG.isTraceEnabled()) {
            LOG.trace(MessageFormat.format("Deleting file (id={0}, path={1}, recursive={2})", //$NON-NLS-1$
                    profile.getId(), stat.getPath(), recursive));
        }
        if (recursive == false && FileSystemCompatibility.isDirectory(stat)) {
            LOG.info(MessageFormat.format("Skip deleting directory (id={0}, path={1})", profile.getId(),
                    stat.getPath()));
        } else {
            counter.add(1);
            succeed &= fs.delete(stat.getPath(), recursive);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish deleting files (id={0}, path={1}, resource={2}, files={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, stats.size()));
    }
    return succeed;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

private List<FileStatus> normalize(List<FileStatus> stats, Path root, Path temporary) {
    assert stats != null;
    assert root != null;
    assert temporary != null;
    List<FileStatus> results = new ArrayList<>();
    for (FileStatus stat : stats) {
        if (root.equals(stat.getPath()) == false && isIn(stat, temporary) == false) {
            results.add(stat);//from   ww  w  .ja v a2  s. com
        }
    }
    return results;
}