Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.alexholmes.hdfsslurper.WorkerThread.java

License:Apache License

private Path getDestPathFromScript(FileStatus srcFile) throws IOException {
    Path p = new Path(ScriptExecutor.getStdOutFromScript(config.getScript(), srcFile.getPath().toString(), 60,
            TimeUnit.SECONDS, config.getFileNameBatchIdDelimiter()));
    String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter();
    String batchId = p.toString().substring(p.toString().lastIndexOf(filenameBatchidDelimiter) + 1,
            p.toString().length());//  ww  w.  ja v  a2  s. co  m
    if (p.toUri().getScheme() == null) {
        throw new IOException("event#Destination path from script must be a URI with a scheme: '" + p + "'"
                + "$batchId#" + batchId);
    }
    return p;
}

From source file:com.aliyun.fs.oss.blk.JetOssFileSystemStore.java

License:Apache License

public static String pathToKey(Path path) {
    if (path.isAbsolute()) {
        // OSS File Path can not start with "/", so we
        // need to scratch the first "/".
        String absolutePath = path.toUri().getPath();
        return absolutePath.substring(1);
    }// ww w.java 2  s .  c  om
    return path.toUri().getPath();
}

From source file:com.aliyun.fs.oss.blk.OssFileSystem.java

License:Apache License

private boolean renameRecursive(Path src, Path dst) throws IOException {
    INode srcINode = store.retrieveINode(src);
    store.storeINode(dst, srcINode);//from  ww  w.j a  va  2  s  . c om
    store.deleteINode(src);
    if (srcINode.isDirectory()) {
        for (Path oldSrc : store.listDeepSubPaths(src)) {
            INode inode = store.retrieveINode(oldSrc);
            if (inode == null) {
                return false;
            }
            String oldSrcPath = oldSrc.toUri().getPath();
            String srcPath = src.toUri().getPath();
            String dstPath = dst.toUri().getPath();
            Path newDst = new Path(oldSrcPath.replaceFirst(srcPath, dstPath));
            store.storeINode(newDst, inode);
            store.deleteINode(oldSrc);
        }
    }
    return true;
}

From source file:com.aliyun.fs.oss.common.InMemoryFileSystemStore.java

License:Apache License

public Set<Path> listDeepSubPaths(Path path) throws IOException {
    Path normalizedPath = normalize(path);
    String pathString = normalizedPath.toUri().getPath();
    if (!pathString.endsWith("/")) {
        pathString += "/";
    }//from   w  ww  . ja  v a  2 s . c  o m
    // This is inefficient but more than adequate for testing purposes.
    Set<Path> subPaths = new LinkedHashSet<Path>();
    for (Path p : inodes.tailMap(normalizedPath).keySet()) {
        if (p.toUri().getPath().startsWith(pathString)) {
            subPaths.add(p);
        }
    }
    return subPaths;
}

From source file:com.aliyun.fs.oss.common.InMemoryFileSystemStore.java

License:Apache License

private Path normalize(Path path) {
    if (!path.isAbsolute()) {
        throw new IllegalArgumentException("Path must be absolute: " + path);
    }//w w w .j a  v a 2  s  .  co m
    return new Path(path.toUri().getPath());
}

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

private static String pathToKey(Path path) {
    if (!path.isAbsolute()) {
        throw new IllegalArgumentException("Path must be absolute: " + path);
    }// w w  w  .jav  a2  s.  c o m
    // OSS File Path can not start with "/", so we need to scratch the first "/".
    String absolutePath = path.toUri().getPath();
    return absolutePath.substring(1);
}

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

/**
 * <p>/*  ww w  .  j  av a 2  s. c o  m*/
 * If <code>f</code> is a file, this method will make a single call to Oss.
 * If <code>f</code> is a directory, this method will make a maximum of
 * (<i>n</i> / 1000) + 2 calls to Oss, where <i>n</i> is the total number of
 * files and directories contained directly in <code>f</code>.
 * </p>
 */
@Override
public FileStatus[] listStatus(Path f) throws IOException {

    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);

    if (key.length() > 0) {
        final FileStatus fileStatus = getFileStatus(f);
        if (fileStatus.isFile()) {
            return new FileStatus[] { fileStatus };
        }
    }

    URI pathUri = absolutePath.toUri();
    Set<FileStatus> status = new TreeSet<FileStatus>();
    String priorLastKey = null;
    do {
        PartialListing listing = store.list(key, OSS_MAX_LISTING_LENGTH, priorLastKey, false);
        for (FileMetadata fileMetadata : listing.getFiles()) {
            Path subPath = keyToPath(fileMetadata.getKey());
            String relativePath = pathUri.relativize(subPath.toUri()).getPath();

            if (fileMetadata.getKey().equals(key + "/")) {
                // this is just the directory we have been asked to list
            } else if (relativePath.endsWith(FOLDER_SUFFIX)) {
                status.add(newDirectory(
                        new Path("/" + relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX)))));
            } else {
                // Here, we need to convert "file/path" to "/file/path".
                // Otherwise, Path.makeQualified will throw `URISyntaxException`.
                Path modifiedPath = new Path("/" + subPath.toString());
                status.add(newFile(fileMetadata, modifiedPath));
            }
        }
        for (String commonPrefix : listing.getCommonPrefixes()) {
            Path subPath = keyToPath(commonPrefix);
            String relativePath = pathUri.relativize(subPath.toUri()).getPath();
            status.add(newDirectory(new Path("/" + relativePath)));
        }
        priorLastKey = listing.getPriorLastKey();
    } while (priorLastKey != null);

    if (status.isEmpty()) {
        return new FileStatus[0];
    }

    return status.toArray(new FileStatus[status.size()]);
}

From source file:com.aliyun.fs.utils.OssInputUtils.java

License:Apache License

public FileSplit[] getSplits(String file, int numSplits) throws IOException {
    Path path = new Path(file);
    this.fs = FileSystem.get(path.toUri(), conf);
    fs.initialize(path.toUri(), conf);//w  ww  .j a  v a  2s. c  om

    FileStatus[] files = fs.listStatus(path);
    long totalSize = 0;
    for (FileStatus file1 : files) {
        if (file1.isDirectory()) {
            throw new IOException("Not a file: " + file1.getPath());
        }
        totalSize += file1.getLen();
    }

    long goalSize = totalSize / (numSplits == 0 ? 1 : numSplits);
    long minSize = Math
            .max(conf.getLong(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.SPLIT_MINSIZE, 1), 1);

    ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
    for (FileStatus file2 : files) {
        Path fp = file2.getPath();
        long length = file2.getLen();
        if (length != 0) {
            long splitSize = Math.max(minSize, goalSize);
            long bytesRemaining = length;
            while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
                FileSplit split = new FileSplit(fp, length - bytesRemaining, splitSize, new String[0]);
                splits.add(split);
                bytesRemaining -= splitSize;
            }
            if (bytesRemaining != 0) {
                FileSplit split = new FileSplit(fp, length - bytesRemaining, bytesRemaining, new String[0]);
                splits.add(split);
            }
        }
    }
    LOG.info("Total # of splits: " + splits.size());
    return splits.toArray(new FileSplit[splits.size()]);
}

From source file:com.aliyun.odps.fs.VolumeFileSystem.java

License:Apache License

@Override
public void setWorkingDirectory(Path new_dir) {
    Path dir = fixRelativePart(new_dir);
    String result = dir.toUri().getPath();
    if (!VolumeFSUtil.isValidName(result)) {
        throw new IllegalArgumentException("Invalid Volume directory name " + result);
    }/*from  w w w .  j a  va  2s . c o m*/
    workingDir = dir;
}

From source file:com.aliyun.odps.fs.VolumeFileSystem.java

License:Apache License

/**
 * Checks that the passed URI belongs to this filesystem and returns just the path component.
 * Expects a URI with an absolute path./*from w w w .  jav  a 2 s.  c  o m*/
 * 
 * @param file URI with absolute path
 * @return path component of {file}
 * @throws IllegalArgumentException if URI does not belong to this DFS
 */
private String getPathName(Path file) {
    checkPath(file);
    String result = file.toUri().getPath();
    if (!VolumeFSUtil.isValidName(result)) {
        throw new IllegalArgumentException(
                "Pathname " + result + " from " + file + " is not a valid VolumeFS filename.");
    }
    return result;
}