List of usage examples for org.apache.hadoop.fs Path isAbsolute
public boolean isAbsolute()
From source file:org.gridgain.grid.ggfs.hadoop.v1.GridGgfsHadoopFileSystem.java
License:Open Source License
/** * Convert Hadoop path into GGFS path.// w ww . j a v a 2 s. co m * * @param path Hadoop path. * @return GGFS path. */ @Nullable private GridGgfsPath convert(@Nullable Path path) { if (path == null) return null; return path.isAbsolute() ? new GridGgfsPath(path.toUri().getPath()) : new GridGgfsPath(convert(workingDir.get()), path.toUri().getPath()); }
From source file:org.gridgain.grid.ggfs.hadoop.v2.GridGgfsHadoopFileSystem.java
License:Open Source License
/** * Convert Hadoop path into GGFS path./*w w w . j ava 2 s . c o m*/ * * @param path Hadoop path. * @return GGFS path. */ @Nullable private GridGgfsPath convert(Path path) { if (path == null) return null; return path.isAbsolute() ? new GridGgfsPath(path.toUri().getPath()) : new GridGgfsPath(workingDir, path.toUri().getPath()); }
From source file:org.kitesdk.data.spi.filesystem.FileSystemUtil.java
License:Apache License
static boolean cleanlyDelete(FileSystem fs, Path root, Path path) { Preconditions.checkNotNull(fs, "File system cannot be null"); Preconditions.checkNotNull(root, "Root path cannot be null"); Preconditions.checkNotNull(path, "Path to delete cannot be null"); try {//from w w w. ja v a2 s .c o m boolean deleted; // attempt to relativize the path to delete Path relativePath; if (path.isAbsolute()) { relativePath = new Path(root.toUri().relativize(path.toUri())); } else { relativePath = path; } if (relativePath.isAbsolute()) { // path is not relative to the root. delete just the path LOG.debug("Deleting path {}", path); deleted = fs.delete(path, true /* include any files */ ); } else { // the is relative to the root path Path absolute = new Path(root, relativePath); LOG.debug("Deleting path {}", absolute); deleted = fs.delete(absolute, true /* include any files */ ); // iterate up to the root, removing empty directories for (Path current = absolute.getParent(); !current.equals(root) && !(current.getParent() == null); current = current.getParent()) { final FileStatus[] stats = fs.listStatus(current); if (stats == null || stats.length == 0) { // dir is empty and should be removed LOG.debug("Deleting empty path {}", current); deleted = fs.delete(current, true) || deleted; } else { // all parent directories will be non-empty break; } } } return deleted; } catch (IOException ex) { throw new DatasetIOException("Could not cleanly delete path:" + path, ex); } }
From source file:org.kitesdk.data.spi.filesystem.TestFileSystemDataset.java
License:Apache License
@Test @SuppressWarnings("deprecation") public void testPathIterator_Partition_Directory() { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash("username", 2).hash("email", 3) .build();// w w w . j a v a2 s . c o m final FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>().namespace("ns") .name("partitioned-users").configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder().schema(USER_SCHEMA).format(format) .compressionType(compressionType).location(testDirectory) .partitionStrategy(partitionStrategy).build()) .type(Record.class).build(); Assert.assertTrue("Dataset is partitioned", ds.getDescriptor().isPartitioned()); Assert.assertEquals(partitionStrategy, ds.getDescriptor().getPartitionStrategy()); writeTestUsers(ds, 10); checkTestUsers(ds, 10); List<Path> dirPaths = Lists.newArrayList(ds.dirIterator()); // 2 user directories * 3 email directories Assert.assertEquals(6, dirPaths.size()); Assert.assertTrue("dirIterator should yield absolute paths.", dirPaths.get(0).isAbsolute()); FileSystemDataset<Record> partition = (FileSystemDataset<Record>) ds.getPartition(new PartitionKey(1, 2), false); List<Path> leafPaths = Lists.newArrayList(partition.dirIterator()); Assert.assertEquals(1, leafPaths.size()); final Path leafPath = leafPaths.get(0); Assert.assertTrue("dirIterator should yield absolute paths.", leafPath.isAbsolute()); Assert.assertEquals(new PartitionKey(1, 2), ds.keyFromDirectory(leafPath)); Assert.assertEquals(new PartitionKey(1), ds.keyFromDirectory(leafPath.getParent())); Assert.assertEquals(new PartitionKey(), ds.keyFromDirectory(leafPath.getParent().getParent())); TestHelpers.assertThrows("Path with too many components", IllegalStateException.class, new Runnable() { @Override public void run() { ds.keyFromDirectory(new Path(leafPath, "extra_dir")); } }); TestHelpers.assertThrows("Non-relative path", IllegalStateException.class, new Runnable() { @Override public void run() { ds.keyFromDirectory(new Path("hdfs://different_host/")); } }); }
From source file:org.oclc.firefly.hadoop.backup.Backup.java
License:Apache License
/** * Entry point//from w w w. j a va 2 s . co m * @param args Command line arguments * @throws Exception exception */ public static void main(String[] args) throws Exception { int initialReplication = 1; int finalReplication = 0; int numMaps = 2; int tries = 0; String tbl = null; String dest = null; String user = System.getProperty("user.name"); Path destPath = null; CommandLineParser parser = new PosixParser(); CommandLine cmdline = null; // Parse command line options try { cmdline = parser.parse(getOptions(), args); } catch (org.apache.commons.cli.ParseException e) { System.out.println(e.getMessage()); printOptions(); System.exit(-1); } // Get command line options for (Option option : cmdline.getOptions()) { switch (option.getId()) { case 'd': dest = option.getValue(); destPath = new Path(dest); if (!destPath.isAbsolute()) { throw new IllegalArgumentException("Destination path must be an absolute path"); } break; case 'm': numMaps = Integer.parseInt(option.getValue()); if (numMaps <= 0) { throw new IllegalArgumentException("Number of map tasks must be greater than zero."); } break; case 'n': tries = Integer.parseInt(option.getValue()); if (tries < 0) { throw new IllegalArgumentException( "Maximum number of tries must be greater than or equal to zero."); } break; case 'f': finalReplication = Integer.parseInt(option.getValue()); if (finalReplication <= 0) { throw new IllegalArgumentException("Initial replication must be greater than zero."); } break; case 'r': initialReplication = Integer.parseInt(option.getValue()); if (initialReplication <= 0) { throw new IllegalArgumentException("Initial replication must be greater than zero."); } break; case 't': tbl = option.getValue(); break; case 'u': user = option.getValue(); break; default: throw new IllegalArgumentException("unexpected option " + option); } } String[] tables = null; if (tbl != null) { tables = tbl.split(","); } Configuration srcConf = HBaseConfiguration.create(); Configuration dstConf = HBaseConfiguration.create(); // This allows us to copy to a separate HDFS instance String destDir = null; if (dest != null) { destDir = destPath.toUri().getPath(); String fsName = null; if (destDir != null && destDir.length() > 0) { LOG.debug("destination dfs: " + dest.substring(0, dest.length() - destDir.length())); fsName = dest.substring(0, dest.length() - destDir.length()); } else { fsName = dest; destDir = null; } if (fsName != null && fsName.length() > 0) { dstConf.set("fs.default.name", fsName); } } Backup backup = new Backup(srcConf, dstConf); backup.setInitialReplication(initialReplication); backup.setFinalReplication(finalReplication); backup.setUsername(user); backup.setNumMapTasks(numMaps); if (destDir != null) { backup.setBackupStoreDirectory(destDir); } LOG.info("HBase backup tool"); LOG.info("--------------------------------------------------"); //LOG.info("Destination fs : " + dstConf.get("fs.default.name")); LOG.info("Initial replication: " + backup.getInitialReplication()); LOG.info("Final replication : " + backup.getFinalReplication()); LOG.info("Number of attempts : " + ((tries == 0) ? "Until nothing left to copy" : tries)); LOG.info("Username : " + backup.getUsername()); LOG.info("Number map tasks : " + backup.getNumMapTasks()); LOG.info("Backup store path : " + backup.getBackupStoreDirectory()); LOG.info("--------------------------------------------------"); boolean success = backup.doMajorCopy(tables, tries); LOG.info("--------------------------------------------------"); if (success) { LOG.info("Backup located at: " + backup.getBackupDirectoryPath()); LOG.info("Backup complete"); } else { LOG.info("Files located at: " + backup.getBackupDirectoryPath()); LOG.info("Backup failed"); } System.exit(success ? 0 : -1); }
From source file:org.shaf.core.io.Location.java
License:Apache License
/** * Constructs a new I/O location.//from w w w.j a va 2 s. co m * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. * @param filter * the filter for selecting files. */ private Location(final FileSystem fs, final Path base, final Path path, final FileNameFilter filter) { if (fs == null) { throw new NullPointerException("file system"); } if (base == null) { throw new NullPointerException("base"); } else if (!base.isAbsolute()) { throw new IllegalArgumentException("The location base-path must be absolute, but received relative: " + base + "; Check the base-path format specified in by the process " + "configuration."); } if (path == null) { throw new NullPointerException("path"); } else if (!path.isAbsolute()) { throw new IllegalArgumentException("The location path must be absolute, but received relative: " + path + "; It is likely that backslash is missing at the beginning" + " of this path."); } if (filter == null) { throw new NullPointerException("filter"); } this.fs = fs; this.base = base; this.path = path; this.filter = filter; this.qualifier = "undefined"; }
From source file:org.shaf.core.io.Location.java
License:Apache License
/** * Resolve the given path against this location. * //from www.ja v a 2 s .c o m * @param others * the paths to resolve against this location. * @return the location with resulting path. */ public final Location resolve(final Path... others) { Path rel = this.path; for (Path other : others) { if (other.isAbsolute()) { rel = Path.mergePaths(rel, other); } else { rel = Path.mergePaths(rel, new Path("/", other)); } } return new Location(this.fs, this.base, rel, this.filter); }
From source file:org.shaf.core.util.IOUtils.java
License:Apache License
/** * Merges paths./*www . j a va 2s . com*/ * * @param paths * the paths to merge. * @return the merged path. * @throws IOException * if an I/O error occurs. */ public static final Path mergePath(final Path... paths) throws IOException { try { if (paths.length == 0) { throw new IllegalArgumentException("paths.length=0"); } if (paths.length == 1) { return paths[0]; } Path buf = null; for (int i = 0; i < paths.length; i++) { if (paths[i] == null) { throw new NullPointerException("paths[" + i + "]"); } Path norm = normalizePath(paths[i]); if (buf == null) { buf = norm; } else { if (norm.isAbsolute()) { norm = new Path(StringUtils.trim(norm.toString(), Path.SEPARATOR)); } buf = new Path(buf, norm); } } return buf; } catch (NullPointerException | IllegalArgumentException exc) { throw new IOException("Invalid input.", exc); } }
From source file:org.shaf.core.util.IOUtils.java
License:Apache License
/** * Merges paths./* w w w .j a v a 2 s .c om*/ * * @param paths * the paths to merge. * @return the merged path. * @throws IOException * if an I/O error occurs. */ public static final Path mergePath(final String... paths) throws IOException { try { if (paths.length == 0) { throw new IllegalArgumentException("paths.length=0"); } Path buf = null; for (int i = 0; i < paths.length; i++) { if (paths[i] == null) { throw new NullPointerException("paths[" + i + "]"); } Path norm = normalizePath(paths[i]); if (buf == null) { buf = norm; } else { if (norm.isAbsolute()) { norm = new Path(StringUtils.trim(norm.toString(), Path.SEPARATOR)); } buf = new Path(buf, norm); } } return buf; } catch (NullPointerException | IllegalArgumentException exc) { throw new IOException("Invalid input.", exc); } }
From source file:org.springframework.data.hadoop.store.AbstractStorage.java
License:Apache License
protected synchronized StreamsHolder<InputStream> getInput(Path inputPath) throws IOException { if (inputHolder == null) { log.info("Creating new InputStream"); inputHolder = new StreamsHolder<InputStream>(); final FileSystem fs = basePath.getFileSystem(configuration); // TODO: hadoop2 isUriPathAbsolute() ? Path p = inputPath.isAbsolute() ? inputPath : new Path(getPath(), inputPath); if (!isCompressed()) { InputStream input = fs.open(p); inputHolder.setStream(input); } else {/*from w w w. ja v a 2 s . c o m*/ Class<?> clazz = ClassUtils.resolveClassName(codecInfo.getCodecClass(), getClass().getClassLoader()); CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz, getConfiguration()); Decompressor decompressor = CodecPool.getDecompressor(compressionCodec); FSDataInputStream winput = fs.open(p); InputStream input = compressionCodec.createInputStream(winput, decompressor); inputHolder.setWrappedStream(winput); inputHolder.setStream(input); } } return inputHolder; }