Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName() 

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.architecting.ch07.MapReduceIndexerTool.java

License:Apache License

private FileStatus[] listSortedOutputShardDirs(Job job, Path outputReduceDir, FileSystem fs)
        throws FileNotFoundException, IOException {
    final String dirPrefix = SolrOutputFormat.getOutputName(job);
    FileStatus[] dirs = fs.listStatus(outputReduceDir, new PathFilter() {
        @Override//from  ww  w  .ja v a2s .  c o m
        public boolean accept(Path path) {
            return path.getName().startsWith(dirPrefix);
        }
    });
    for (FileStatus dir : dirs) {
        if (!dir.isDirectory()) {
            throw new IllegalStateException("Not a directory: " + dir.getPath());
        }
    }

    // use alphanumeric sort (rather than lexicographical sort) to properly handle more than 99999
    // shards
    Arrays.sort(dirs, new Comparator<FileStatus>() {
        @Override
        public int compare(FileStatus f1, FileStatus f2) {
            return new AlphaNumericComparator().compare(f1.getPath().getName(), f2.getPath().getName());
        }
    });

    return dirs;
}

---

From source file:com.asakusafw.bulkloader.collector.ExportFileSend.java

License:Apache License

/**
 * ?Hadoop??????{@code true}?/*from   w  w w .ja v  a  2  s. c o m*/
 * @param path ?
 * @return ???????{@code true}????????{@code false}
 */
private boolean isSystemFile(Path path) {
    assert path != null;
    String name = path.getName();
    return name.equals(FileOutputCommitter.SUCCEEDED_FILE_NAME) || name.equals("_logs");
}

---

From source file:com.asakusafw.cleaner.main.HDFSCleaner.java

License:Apache License

/**
 * ?/*from  w  ww  .j  av  a  2s.c  om*/
 * @param fs HDFS?
 * @param cleanPath HDFS??
 * @param isSetExecutionId ID????????
 * @param pattern 
 * @param keepDate ??
 * @param now ?
 * @param recursive ????
 * @return ?
 * @throws CleanerSystemException
 */
private boolean cleanDir(FileSystem fs, Path cleanPath, boolean isSetExecutionId, String pattern, int keepDate,
        Date now, boolean recursive) throws CleanerSystemException {
    try {
        if (!fs.exists(cleanPath)) {
            // ??????
            Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                    "??????", cleanPath.toString());
            return false;
        }
        if (!fs.getFileStatus(cleanPath).isDir()) {
            // ??????
            Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                    "??????", cleanPath.toString());
            return false;
        }

        // ?
        Log.log(CLASS, MessageIdConst.HCLN_FILE_DELETE, cleanPath.toString());
        int cleanFileCount = 0;
        int cleanDirCount = 0;
        boolean result = true;
        FileStatus[] dirStatus = getListStatus(fs, cleanPath);
        Path[] listedPaths = FileUtil.stat2Paths(dirStatus);
        for (Path path : listedPaths) {
            FileStatus status = fs.getFileStatus(path);
            long lastModifiedTime = status.getModificationTime();
            if (status.isDir() && recursive) {
                // ????????
                if (isSetExecutionId) {
                    // ID??????MM???????
                    String executionId = path.getName();
                    if (isRunningJobFlow(executionId)) {
                        // ???????
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_EXEC, path.toString());
                        continue;
                    }
                }
                FileStatus[] childdirStatus = getListStatus(fs, path);
                if (childdirStatus.length == 0) {
                    // ???????
                    if (isExpired(lastModifiedTime, keepDate, now)) {
                        if (!fs.delete(path, false)) {
                            Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "",
                                    path.toString());
                            result = false;
                        } else {
                            cleanDirCount++;
                            Log.log(CLASS, MessageIdConst.HCLN_DIR_DELETE, path.toString());
                        }
                    }
                } else {
                    // ?????????
                    if (cleanDir(fs, path, false, pattern, keepDate, now, recursive)) {
                        // ????????
                        childdirStatus = getListStatus(fs, path);
                        if (childdirStatus.length == 0) {
                            if (isExpired(lastModifiedTime, keepDate, now)) {
                                if (!fs.delete(path, false)) {
                                    Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "",
                                            path.toString());
                                    result = false;
                                } else {
                                    cleanDirCount++;
                                    Log.log(CLASS, MessageIdConst.HCLN_DIR_DELETE, path.toString());
                                }
                            }
                        }
                    } else {
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "", path.toString());
                        result = false;
                    }
                }
            } else if (!status.isDir()) {
                // ???????????
                if (isExpired(lastModifiedTime, keepDate, now) && isMatchPattern(path, pattern)) {
                    if (!fs.delete(path, false)) {
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "", path.toString());
                        result = false;
                    } else {
                        Log.log(CLASS, MessageIdConst.HCLN_DELETE_FILE, path.toString());
                        cleanFileCount++;
                    }
                }
            }
        }

        Log.log(CLASS, MessageIdConst.HCLN_FILE_DELETE_SUCCESS, cleanPath.toString(), cleanDirCount,
                cleanFileCount);

        return result;
    } catch (IOException e) {
        Log.log(e, CLASS, MessageIdConst.HCLN_CLEN_DIR_EXCEPTION, cleanPath.getName());
        return false;
    }
}

---

From source file:com.asakusafw.compiler.util.tester.HadoopDriver.java

License:Apache License

private void copyFromHadoop(Location location, File targetDirectory) throws IOException {
    targetDirectory.mkdirs();//from www. j ava2  s .c o m
    logger.info("copy {} to {}", location, targetDirectory);

    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(configuration);
    FileStatus[] list = fs.globStatus(path);
    if (list == null) {
        throw new IOException(
                MessageFormat.format("Failed to fs -get: source={0}, destination={1}", path, targetDirectory));
    }
    for (FileStatus status : list) {
        Path p = status.getPath();
        try {
            fs.copyToLocalFile(p, new Path(new File(targetDirectory, p.getName()).toURI()));
        } catch (IOException e) {
            throw new IOException(
                    MessageFormat.format("Failed to fs -get: source={0}, destination={1}", p, targetDirectory),
                    e);
        }
    }
}

---

From source file:com.asakusafw.operation.tools.directio.file.FilePutCommand.java

License:Apache License

private void copyOnto(List<java.nio.file.Path> sources, org.apache.hadoop.fs.Path destination) {
    sources.stream().filter(it -> recursive || Files.isDirectory(it) == false)
            .collect(Collectors.groupingBy(it -> Optional.ofNullable(it.getFileName()).map(String::valueOf)
                    .orElseThrow(() -> new IllegalStateException(it.toString()))))
            .forEach((k, v) -> {/*from  w w w.j a  v  a  2 s  . c  o  m*/
                org.apache.hadoop.fs.Path dst = resolve(destination, k);
                if (v.size() >= 2) {
                    throw new CommandConfigurationException(
                            MessageFormat.format("conflict destination file \"{0}\": {1}", dst,
                                    v.stream().map(String::valueOf).collect(Collectors.joining(", "))));
                }
                java.nio.file.Path src = v.get(0);
                if (overwriteParameter.isEnabled() == false && stat(dst).isPresent()) {
                    throw new CommandConfigurationException(
                            MessageFormat.format("destination file already exists: {0} ({1})", dst, src));
                }
            });
    try (PrintWriter writer = outputParameter.open()) {
        executorParameter.execute(sources.stream().map(source -> {
            org.apache.hadoop.fs.Path src = asHadoopPath(source);
            org.apache.hadoop.fs.Path dst = resolve(destination, src.getName());
            return new Copy(writer, dataSourceParameter.getHadoopFileSystem(src), src,
                    dataSourceParameter.getHadoopFileSystem(dst), dst);
        }).collect(Collectors.toList()));
    }
}

---

From source file:com.asakusafw.operation.tools.directio.file.Util.java

License:Apache License

private static org.apache.hadoop.fs.Path normalize(org.apache.hadoop.fs.Path path) {
    if (path.getName().isEmpty()) {
        return Optional.ofNullable(path.getParent()).orElse(path);
    }/*from w  w  w.  jav  a  2  s.c  om*/
    return path;
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static String getMarkPath(Path path, Pattern pattern) {
    assert path != null;
    assert pattern != null;
    String name = path.getName();
    Matcher matcher = pattern.matcher(name);
    if (matcher.matches() == false) {
        return null;
    }/*from   ww  w  .j  a  v a2s  . c o  m*/
    return matcher.group(1);
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

private Path findCacheForLocalMode(String resourceName, String localName) throws IOException {
    assert resourceName != null;
    assert localName != null;
    Path remotePath = null;/* www.j a  v a  2s  . c om*/
    String remoteName = null;
    for (URI uri : DistributedCache.getCacheFiles(configuration)) {
        if (localName.equals(uri.getFragment())) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("fragment matched: " + uri); //$NON-NLS-1$
            }
            String rpath = uri.getPath();
            remotePath = new Path(uri);
            remoteName = rpath.substring(rpath.lastIndexOf('/') + 1);
            break;
        }
    }
    if (remoteName == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("fragment not matched: " + resourceName); //$NON-NLS-1$
        }
        return null;
    }
    assert remotePath != null;
    for (Path path : getLocalCacheFiles()) {
        String localFileName = path.getName();
        if (remoteName.equals(localFileName) == false) {
            continue;
        }
        if (localFileSystem.exists(path) == false) {
            continue;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("local path matched: " + path); //$NON-NLS-1$
        }
        return localFileSystem.makeQualified(path);
    }
    FileSystem remoteFileSystem = remotePath.getFileSystem(configuration);
    remotePath = remoteFileSystem.makeQualified(remotePath);
    if (LOG.isDebugEnabled()) {
        LOG.debug("distributed cache is not localized explicitly: " + remotePath); //$NON-NLS-1$
    }
    if (isLocal(remoteFileSystem) == false) {
        LOG.warn(MessageFormat.format("Failed to resolve stage resource in local cache \"{1}\" (resource={0})",
                resourceName, localName));
    }
    return remotePath;
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path computeCachePath(Path file) {
    assert repository != null;
    String directoryName;/*from   ww w  . ja  va 2s . c o m*/
    Path parent = file.getParent();
    if (parent == null) {
        directoryName = String.format("%08x", 0); //$NON-NLS-1$
    } else {
        directoryName = String.format("%08x", parent.toString().hashCode()); //$NON-NLS-1$
    }
    Path directory = new Path(repository, directoryName);
    Path target = new Path(directory, file.getName());
    return target;
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path computeCacheChecksumPath(Path cachePath) {
    Path parent = cachePath.getParent();
    String name = String.format("%s.acrc", cachePath.getName()); //$NON-NLS-1$
    return new Path(parent, name);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »