Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.netflix.bdp.s3mper.listing.ConsistentListingAspectTest.java

License:Apache License

@Test
public void testUpdateMetastore() throws Throwable {
    System.out.println("updateMetastore");
    Path arg1Path = new Path(testPath + "/update.test");
    OutputStream fout = deleteFs.create(arg1Path);
    assertNotNull(fout);//w  ww . j  av a  2s.  co m
    fout.close();
    List<FileInfo> files = meta.list(Collections.singletonList(arg1Path.getParent()));
    assertEquals(1, files.size());
    deleteFs.delete(arg1Path, true);
    janitor.clearPath(testPath);
}

From source file:com.netflix.bdp.s3mper.listing.DarkloadTest.java

License:Apache License

@Test
public void testDarkloading() throws Throwable {
    Path path = new Path(testPath + "/test");
    meta.add(path, false);//from   w w w. j  a v  a  2s .c  o  m
    assertEquals(0, fs.listStatus(path.getParent()).length);
}

From source file:com.netflix.bdp.s3mper.listing.DarkloadTest.java

License:Apache License

@Test(expected = FileNotFoundException.class)
public void testDarkloadingDisabled() throws Throwable {
    Configuration noDarkloadConf = new Configuration(conf);
    noDarkloadConf.setBoolean("s3mper.failOnError", true);
    noDarkloadConf.setBoolean("s3mper.darkload", false);
    FileSystem noDarkloadFs = FileSystem.get(testPath.toUri(), noDarkloadConf);
    Path path = new Path(testPath + "/test");
    meta.add(path, false);//from  ww w.  j a  v  a2 s .  c  om
    noDarkloadFs.listStatus(path.getParent());
}

From source file:com.netflix.bdp.s3mper.metastore.impl.InMemoryMetastore.java

License:Apache License

@Override
public void add(Path path, boolean directory) throws Exception {
    synchronized (this) {
        delete(path);/* w  w  w  . jav a  2s. c  om*/
        get(path.getParent()).add(new FileInfo(path, false, directory));
    }
}

From source file:com.netflix.bdp.s3mper.metastore.impl.InMemoryMetastore.java

License:Apache License

@Override
public void delete(final Path path) throws Exception {
    synchronized (this) {
        List<FileInfo> list = get(path.getParent());
        for (Iterator<FileInfo> it = list.iterator(); it.hasNext();) {
            FileInfo fi = it.next();/*  w ww.j  a va2  s  .  com*/
            if (fi.getPath().equals(path)) {
                it.remove();
            }
        }
    }
}

From source file:com.newland.bi.bigdata.hdfs.Configuration.java

License:Apache License

/** 
 * Get a local file under a directory named by <i>dirsProp</i> with
 * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
 * then one is chosen based on <i>path</i>'s hash code.  If the selected
 * directory does not exist, an attempt is made to create it.
 * //  w ww  .  j  a v a  2s  .co  m
 * @param dirsProp directory in which to locate the file.
 * @param path file-path.
 * @return local file under the directory with the given path.
 */
public Path getLocalPath(String dirsProp, String path) throws IOException {
    String[] dirs = getTrimmedStrings(dirsProp);
    int hashCode = path.hashCode();
    FileSystem fs = null; //FileSystem.getLocal(this);
    for (int i = 0; i < dirs.length; i++) { // try each local dir
        int index = (hashCode + i & Integer.MAX_VALUE) % dirs.length;
        Path file = new Path(dirs[index], path);
        Path dir = file.getParent();
        if (fs.mkdirs(dir) || fs.exists(dir)) {
            return file;
        }
    }
    LOG.warn("Could not make " + path + " in local directories from " + dirsProp);
    for (int i = 0; i < dirs.length; i++) {
        int index = (hashCode + i & Integer.MAX_VALUE) % dirs.length;
        LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
    }
    throw new IOException("No valid local directories in property: " + dirsProp);
}

From source file:com.ngdata.hbaseindexer.mr.HBaseIndexingOptionsTest.java

License:Apache License

@Test
public void testEvaluateOutputDir_GoLive() {
    opts.outputDir = null;/*w  ww.j av  a2s.c o  m*/
    opts.reducers = 2;
    opts.goLive = true;

    opts.evaluateOutputDir();

    Path outputPath = opts.outputDir;
    assertEquals(new Path("/tmp"), outputPath.getParent());
    assertTrue(opts.isGeneratedOutputDir());
}

From source file:com.ngdata.hbaseindexer.mr.HBaseIndexingOptionsTest.java

License:Apache License

@Test
public void testEvaluateOutputDir_GoLive_AlternateTempDirViaConfig() {
    opts.outputDir = null;//from  w  ww .jav  a2s.  c o  m
    opts.reducers = 2;
    opts.goLive = true;
    conf.set("hbase.search.mr.tmpdir", "/othertmp");

    opts.evaluateOutputDir();

    Path outputPath = opts.outputDir;
    assertEquals(new Path("/othertmp"), outputPath.getParent());
    assertTrue(opts.isGeneratedOutputDir());
}

From source file:com.ning.metrics.action.hdfs.reader.HdfsListing.java

License:Apache License

public HdfsListing(FileSystem fileSystem, Path path, boolean raw,
        RowFileContentsIteratorFactory rowFileContentsIteratorFactory, boolean recursive) throws IOException {
    this.path = path;
    this.parentPath = "/".equals(path.toUri().toString()) ? null : path.getParent().toUri().toString();
    this.raw = raw;
    this.recursive = recursive;
    this.rowFileContentsIteratorFactory = rowFileContentsIteratorFactory;

    final ImmutableList.Builder<HdfsEntry> entriesBuilder = ImmutableList.builder();
    findEntries(fileSystem, path, entriesBuilder);
    this.entries = entriesBuilder.build();
}

From source file:com.packetloop.packetpig.storage.JsonMetadata.java

License:Apache License

/**.
 * Given a path, which may represent a glob pattern, a directory,
 * comma separated files/glob patterns or a file, this method
 * finds the set of relevant metadata files on the storage system.
 * The algorithm for finding the metadata file is as follows:
 * <p>/*from  w  w  w .  j  a v  a 2 s.  c o  m*/
 * For each object represented by the path (either directly, or via a glob):
 *   If object is a directory, and path/metaname exists, use that as the metadata file.
 *   Else if parentPath/metaname exists, use that as the metadata file.
 * <p>
 * Resolving conflicts, merging the metadata, etc, is not handled by this method and should be
 * taken care of by downstream code.
 * <p>
 * @param path      Path, as passed in to a LoadFunc (may be a Hadoop glob)
 * @param metaname    Metadata file designation, such as .pig_schema or .pig_stats
 * @param conf      configuration object
 * @return Set of element descriptors for all metadata files associated with the files on the path.
 */
protected Set<ElementDescriptor> findMetaFile(String path, String metaname, Configuration conf)
        throws IOException {
    Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
    String[] locations = LoadFunc.getPathStrings(path);
    for (String loc : locations) {
        DataStorage storage;

        storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));

        String fullPath = FileLocalizer.fullPath(loc, storage);

        if (storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, metaname);
            if (exists(metaFilePath)) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(loc);
            for (ElementDescriptor descriptor : descriptors) {
                ContainerDescriptor container = null;

                if (descriptor instanceof HFile) {
                    Path descriptorPath = ((HPath) descriptor).getPath();
                    String fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    String parentName = parent.toString();
                    container = new HDirectory((HDataStorage) storage, parent);
                } else { // descriptor instanceof HDirectory
                    container = (HDirectory) descriptor;
                }

                // if no custom schema, try the parent directory
                ElementDescriptor metaFilePath = storage.asElement(container, metaname);
                if (exists(metaFilePath)) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
    }
    return metaFileSet;
}