List of usage examples for org.apache.hadoop.fs Path isAbsolute
public boolean isAbsolute()
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * Turns a path (relative or otherwise) into an COS key * * @host hostName host of the object//w w w . j av a 2 s. c o m * @param path object full path */ private String pathToKey(String hostName, Path path) { if (!path.isAbsolute()) { String pathStr = path.toUri().getPath(); if (pathStr.startsWith(mBucket) && !pathStr.equals(mBucket)) { path = new Path(pathStr.substring(mBucket.length() + 1)); } path = new Path(hostName, path); } if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) { return ""; } return path.toUri().getPath().substring(1); }
From source file:com.kadwa.hadoop.DistExec.java
License:Open Source License
/** * Make a path relative with respect to a root path. * absPath is always assumed to descend from root. * Otherwise returned path is null./*from w ww . j a va 2 s . c o m*/ */ static String makeRelative(Path root, Path absPath) { if (!absPath.isAbsolute()) { throw new IllegalArgumentException("!absPath.isAbsolute(), absPath=" + absPath); } String p = absPath.toUri().getPath(); StringTokenizer pathTokens = new StringTokenizer(p, "/"); for (StringTokenizer rootTokens = new StringTokenizer(root.toUri().getPath(), "/"); rootTokens .hasMoreTokens();) { if (!rootTokens.nextToken().equals(pathTokens.nextToken())) { return null; } } StringBuilder sb = new StringBuilder(); for (; pathTokens.hasMoreTokens();) { sb.append(pathTokens.nextToken()); if (pathTokens.hasMoreTokens()) { sb.append(Path.SEPARATOR); } } return sb.length() == 0 ? "." : sb.toString(); }
From source file:com.liveramp.hank.storage.HdfsPartitionRemoteFileOps.java
License:Apache License
public HdfsPartitionRemoteFileOps(String remoteDomainRoot, int partitionNumber, CompressionCodec compressionCodec, boolean useTrash) throws IOException { this.useTrash = useTrash; this.partitionRoot = remoteDomainRoot + "/" + partitionNumber; Path partitionRootPath = new Path(partitionRoot); this.fs = FileSystemHelper.getFileSystemForPath(remoteDomainRoot); if (!partitionRootPath.isAbsolute()) { throw new IOException("Cannot initialize " + this.getClass().getSimpleName() + " with a non absolute remote partition root: " + partitionRoot); }//from ww w. j a va2 s. c om this.compressionCodec = compressionCodec; }
From source file:com.netflix.aegisthus.tools.Utils.java
License:Apache License
public static void copy(Path from, Path to, boolean snappy, TaskAttemptContext ctx) throws IOException { FileSystem fromFs = from.getFileSystem(ctx.getConfiguration()); FileSystem toFs = to.getFileSystem(ctx.getConfiguration()); if (!to.isAbsolute()) { to = new Path(ctx.getConfiguration().get("mapred.working.dir"), to); }//from ww w . jav a2 s . c om if (!snappy && onSameHdfs(ctx.getConfiguration(), from, to)) { LOG.info(String.format("renaming %s to %s", from, to)); toFs.mkdirs(to.getParent()); toFs.rename(from, to); return; } InputStream in = fromFs.open(from); OutputStream out = toFs.create(to, false); try { if (snappy) { in = new SnappyInputStream2(in); } byte[] buffer = new byte[65536]; int bytesRead; int count = 0; while ((bytesRead = in.read(buffer)) >= 0) { if (bytesRead > 0) { out.write(buffer, 0, bytesRead); } if (count++ % 50 == 0) { ctx.progress(); } } } finally { in.close(); out.close(); } }
From source file:com.quantcast.qfs.hadoop.QuantcastFileSystem.java
License:Apache License
protected Path makeAbsolute(Path path) throws IOException { if (path.isAbsolute()) { checkPath(path);//from w w w.ja v a 2 s . c om return path; } if (null == workingDir) { throw new IOException(path + ": absolute path required"); } return new Path(workingDir, path); }
From source file:com.quixey.hadoop.fs.oss.OSSFileSystem.java
License:Apache License
/** * Constructs a OSS key from given {@code path}. * * If a path has a trailing slash, it's removed. * * @param path absolute HDFS path//from w w w . jav a 2 s . c om * @return OSS key */ private static String pathToKey(Path path) { if (null != path.toUri().getScheme() && path.toUri().getPath().isEmpty()) { // allow uris without trailing slash after bucket to refer to root, // like oss://mybucket return ""; } checkArgument(path.isAbsolute(), "Path must be absolute: " + path); String ret = path.toUri().getPath().substring(1); // remove initial slash if (ret.endsWith("/") && ret.indexOf("/") != ret.length() - 1) { ret = ret.substring(0, ret.length() - 1); } return ret; }
From source file:com.quixey.hadoop.fs.oss.OSSFileSystem.java
License:Apache License
/** * @return absolute path, from {@code path}. *//*w w w . ja v a 2 s. co m*/ private Path makeAbsolute(Path path) { if (path.isAbsolute()) return path; // assume that all non-absolute paths are relative to the working directory return new Path(workingDir, path); }
From source file:com.rapleaf.hank.storage.HdfsPartitionRemoteFileOps.java
License:Apache License
public HdfsPartitionRemoteFileOps(String remoteDomainRoot, int partitionNumber, CompressionCodec compressionCodec) throws IOException { this.partitionRoot = remoteDomainRoot + "/" + partitionNumber; Path partitionRootPath = new Path(partitionRoot); this.fs = FileSystem.get(new Configuration()); if (!partitionRootPath.isAbsolute()) { throw new IOException("Cannot initialize " + this.getClass().getSimpleName() + " with a non absolute remote partition root: " + partitionRoot); }// www . j a v a 2s . co m this.compressionCodec = compressionCodec; }
From source file:com.rapleaf.ramhdfs.RamFileSystem.java
License:Apache License
private static Path makeAbsolute(Path f) { if (f.isAbsolute()) { return new Path("ram:" + f.toUri().getSchemeSpecificPart()); } else {/* w ww.ja v a2s. c o m*/ return new Path(workingDir, f); } }
From source file:com.thinkbiganalytics.nifi.v2.hdfs.RemoveHDFSFolder.java
License:Apache License
@Override public void onTrigger(@Nonnull final ProcessContext context, @Nonnull final ProcessSession session) throws ProcessException { // Get file to process FlowFile flowFile = session.get();//w w w . ja v a 2 s .com if (flowFile == null) { return; } // Get file system FileSystem fileSystem = getFileSystem(context); if (fileSystem == null) { session.transfer(flowFile, REL_FAILURE); return; } // Delete the specified paths String[] directories = context.getProperty(DIRECTORY).evaluateAttributeExpressions(flowFile).getValue() .split("\\r?\\n"); for (String string : directories) { // Check for possible missing properties - accidentally deleting parent directory instead of child String pathString = string.trim(); if (!pathString.endsWith("/")) { getLog().error("Path must end with a slash /: " + pathString); session.transfer(flowFile, REL_FAILURE); return; } if (pathString.contains("//")) { getLog().error("Path cannot contain double slashes //: " + pathString); session.transfer(flowFile, REL_FAILURE); return; } // Check for relative directories - accidentally deleting folder in home directory Path path = new Path(pathString); if (!path.isAbsolute()) { getLog().error("Path is not absolute: " + path); session.transfer(flowFile, REL_FAILURE); return; } // Delete path getLog().debug("Deleting path: " + path); try { if (!fileSystem.delete(path, true) && fileSystem.exists(path)) { getLog().error("Failed to remove path: " + path); session.transfer(flowFile, REL_FAILURE); return; } } catch (IOException e) { getLog().error("Failed to remove path: " + path, e); session.transfer(flowFile, REL_FAILURE); return; } } // Return success session.transfer(flowFile, REL_SUCCESS); }