Example usage for org.apache.hadoop.fs FileStatus toString

List of usage examples for org.apache.hadoop.fs FileStatus toString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus toString.

Prototype

@Override
    public String toString()

Source Link

Usage

From source file:com.fullcontact.sstable.hadoop.mapreduce.SSTableInputFormat.java

License:Apache License

@Override
protected List<FileStatus> listStatus(final JobContext job) throws IOException {

    final List<FileStatus> files = Lists.newArrayList();

    for (FileStatus file : super.listStatus(job)) {
        files.addAll(handleFile(file, job));
    }//  ww w.  jav  a2s .  com

    LOG.debug("Initial file list: {} {}", files.size(), files);

    final Configuration configuration = job.getConfiguration();

    for (Iterator<FileStatus> iterator = files.iterator(); iterator.hasNext();) {
        final FileStatus fileStatus = iterator.next();
        final Path file = fileStatus.getPath();
        final FileSystem fs = file.getFileSystem(configuration);

        if (!SSTablePredicates.IS_SSTABLE.apply(file.toString())) {
            // Ignore non-sstable date files, always (for now)
            LOG.debug("Removing non-sstable file: {}", file);
            iterator.remove();
        } else {
            // read the index file
            LOG.debug("Reading index file for sstable file: {}", file);

            final Path indexFile = SSTableFunctions.INDEX_FILE.apply(file);

            LOG.debug("Reading index file: {}", indexFile);

            final SSTableIndexIndex index = SSTableIndexIndex.readIndex(fs, indexFile);
            indexes.put(file, index);
        }
    }

    LOG.debug("Final file list: {} {}", files.size(), files);

    return files;
}

From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java

License:Open Source License

/**
 * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
 * @param fs filesystem//from   w  w  w  . ja v  a2s. c  o m
 * @param dir directory to scan
 * @param subdir full path to look for
 * @throws IOException IO problems
 */
public static void assertListStatusFinds(FileSystem fs, Path dir, Path subdir) throws IOException {
    FileStatus[] stats = fs.listStatus(dir);
    boolean found = false;
    StringBuilder builder = new StringBuilder();
    for (FileStatus stat : stats) {
        builder.append(stat.toString()).append('\n');
        if (stat.getPath().equals(subdir)) {
            found = true;
        }
    }
    assertTrue("Path " + subdir + " not found in directory " + dir + ":" + builder, found);
}

From source file:com.ibm.stocator.fs.swift2d.systemtests.TestSwiftFileSystemLsOperations.java

License:Apache License

@Ignore("Not supported")
public void testListStatusFiltered() throws Throwable {
    Path dir = path(getBaseURI() + "/");
    Path child = path(getBaseURI() + "/test");
    touch(sFileSystem, child);//from www . ja v a 2s  .  c o  m
    FileStatus[] stats = sFileSystem.listStatus(dir, new AcceptAllFilter());
    boolean found = false;
    StringBuilder builder = new StringBuilder();
    for (FileStatus stat : stats) {
        builder.append(stat.toString()).append('\n');
        if (stat.getPath().equals(child)) {
            found = true;
        }
    }
    assertTrue("Path " + child + " not found in directory " + dir + ":" + builder, found);
}

From source file:com.wandisco.s3hdfs.rewrite.filter.TestFailureCases.java

License:Apache License

@Test
public void testDeleteObjectMultipleTimesWithVersioning() throws Exception {
    S3HdfsPath s3HdfsPath = testUtil.setUpS3HdfsPath("myBucket", "bigFile");

    // Enable versioning
    S3Bucket bucket = s3Service.createBucket(new S3Bucket(s3HdfsPath.getBucketName()));

    s3Service.enableBucketVersioning(bucket.getName());

    // Prepare the object and bucket
    String objectKey = s3HdfsPath.getObjectName();

    // Put new object
    byte[] data = new byte[SMALL_SIZE];
    for (int i = 0; i < SMALL_SIZE; i++) {
        data[i] = (byte) (i % 256);
    }//from   w ww  . j  a  va  2  s  . c  om
    S3Object object = new S3Object(objectKey, data);
    s3Service.putObject(bucket.getName(), object);

    // These should create delete markers.
    s3Service.deleteObject(bucket.getName(), object.getName());
    s3Service.deleteObject(bucket.getName(), object.getName());
    s3Service.deleteObject(bucket.getName(), object.getName());

    FileStatus[] fs = hdfs.listStatus(new Path(s3HdfsPath.getHdfsRootObjectPath()));

    for (FileStatus f : fs) {
        System.out.println(f.toString());
    }

    assertEquals(4, fs.length);
}

From source file:org.apache.hoya.avro.RoleHistoryWriter.java

License:Apache License

/**
 * Find all history entries in a dir. The dir is created if it is
 * not already defined.//from w w w .  ja  v a2s  .co m
 * 
 * The scan uses the match pattern {@link HoyaKeys#HISTORY_FILENAME_MATCH_PATTERN}
 * while dropping empty files and directories which match the pattern.
 * The list is then sorted with a comparator that sorts on filename,
 * relying on the filename of newer created files being later than the old ones.
 * 
 * 
 *
 * @param fs filesystem
 * @param dir dir to scan
 * @param includeEmptyFiles should empty files be included in the result?
 * @return a possibly empty list
 * @throws IOException IO problems
 * @throws FileNotFoundException if the target dir is actually a path
 */
public List<Path> findAllHistoryEntries(FileSystem fs, Path dir, boolean includeEmptyFiles) throws IOException {
    assert fs != null;
    assert dir != null;
    if (!fs.exists(dir)) {
        fs.mkdirs(dir);
    } else if (!fs.isDirectory(dir)) {
        throw new FileNotFoundException("Not a directory " + dir.toString());
    }

    PathFilter filter = new GlobFilter(HoyaKeys.HISTORY_FILENAME_GLOB_PATTERN);
    FileStatus[] stats = fs.listStatus(dir, filter);
    List<Path> paths = new ArrayList<Path>(stats.length);
    for (FileStatus stat : stats) {
        log.debug("Possible entry: {}", stat.toString());
        if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {
            paths.add(stat.getPath());
        }
    }
    sortHistoryPaths(paths);
    return paths;
}

From source file:org.apache.sentry.tests.e2e.hdfs.TestDbHdfsBase.java

License:Apache License

@After
public void clearAfterPerTest() throws Exception {
    super.clearAfterPerTest();
    // Clean up any extra data created during testing in external path
    LOGGER.info("TestDbHdfsBase clearAfterPerTest");
    kinitFromKeytabFile(dfsAdmin, getKeyTabFileFullPath(dfsAdmin));
    if (!Strings.isNullOrEmpty(testExtPathDir)) {
        Path path = new Path(testExtPathDir);
        FileStatus[] children = fileSystem.listStatus(path);
        for (FileStatus fs : children) {
            LOGGER.info("Deleting " + fs.toString());
            fileSystem.delete(fs.getPath(), true);
        }/*w  w  w. j a  v  a 2  s  .  com*/
    }
}

From source file:org.apache.slider.server.avro.RoleHistoryWriter.java

License:Apache License

/**
 * Find all history entries in a dir. The dir is created if it is
 * not already defined./*from   w  ww . ja va2  s.c  o  m*/
 * 
 * The scan uses the match pattern {@link SliderKeys#HISTORY_FILENAME_MATCH_PATTERN}
 * while dropping empty files and directories which match the pattern.
 * The list is then sorted with a comparator that sorts on filename,
 * relying on the filename of newer created files being later than the old ones.
 * 
 * 
 *
 * @param fs filesystem
 * @param dir dir to scan
 * @param includeEmptyFiles should empty files be included in the result?
 * @return a possibly empty list
 * @throws IOException IO problems
 * @throws FileNotFoundException if the target dir is actually a path
 */
public List<Path> findAllHistoryEntries(FileSystem fs, Path dir, boolean includeEmptyFiles) throws IOException {
    assert fs != null;
    assert dir != null;
    if (!fs.exists(dir)) {
        fs.mkdirs(dir);
    } else if (!fs.isDirectory(dir)) {
        throw new FileNotFoundException("Not a directory " + dir.toString());
    }

    PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN);
    FileStatus[] stats = fs.listStatus(dir, filter);
    List<Path> paths = new ArrayList<Path>(stats.length);
    for (FileStatus stat : stats) {
        log.debug("Possible entry: {}", stat.toString());
        if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {
            paths.add(stat.getPath());
        }
    }
    sortHistoryPaths(paths);
    return paths;
}

From source file:org.apache.slider.test.ContractTestUtils.java

License:Apache License

/**
 * Read a file, verify its length and contents match the expected array
 * @param fs filesystem/*  w  ww. java2 s.co  m*/
 * @param path path to file
 * @param original original dataset
 * @throws IOException IO Problems
 */
public static void verifyFileContents(FileSystem fs, Path path, byte[] original) throws IOException {
    FileStatus stat = fs.getFileStatus(path);
    String statText = stat.toString();
    assertTrue("not a file " + statText, stat.isFile());
    assertEquals("wrong length " + statText, original.length, stat.getLen());
    byte[] bytes = readDataset(fs, path, original.length);
    compareByteArrays(original, bytes, original.length);
}

From source file:org.bgi.flexlab.gaea.data.mapreduce.util.HdfsFilesReader.java

License:Open Source License

public void traversal(String path, PathFilter pathFilter) {
    Path p = new Path(path);

    fs = HdfsFileManager.getFileSystem(p, conf);
    FileStatus status = null;/*from w  w  w  .j  a va  2s  . c  o  m*/
    try {
        status = fs.getFileStatus(p);
    } catch (IOException e2) {
        throw new FileNotExistException(p.getName());
    }

    if (status.isFile()) {
        if (!filter(p.getName()))
            files.add(p);
    } else {
        FileStatus[] stats = null;
        try {
            stats = fs.listStatus(p, pathFilter);
        } catch (IOException e) {
            throw new RuntimeException(e.toString());
        }

        for (FileStatus file : stats) {
            if (!file.isFile()) {
                traversal(file.toString(), pathFilter);
            } else {
                if (!filter(file.getPath().getName()))
                    files.add(file.getPath());
            }
        }
    }

    if (size() == 0)
        return;
    FSDataInputStream currInput;
    try {
        currInput = fs.open(files.get(0));
        lineReader = new LineReader(currInput, conf);
    } catch (IOException e) {
        throw new RuntimeException(e.toString());
    }
}

From source file:org.mrgeo.gce.MrGeoFormatFactorySpi.java

License:Open Source License

public MrGeoFormatFactorySpi() {
    try {/*from   w ww. j a v  a2 s. c  om*/
        //LoggingUtils.setLogLevel("org.mrgeo", LoggingUtils.DEBUG);
        if (log.isLoggable(Level.FINE)) {
            try {
                log.fine("MrGeo environment:");
                String env = System.getenv("MRGEO_HOME");
                log.fine("  MRGEO_HOME: " + ((env == null) ? "null" : env));
                env = System.getenv("HADOOP_CONF_DIR");
                log.fine("  HADOOP_CONF_DIR: " + ((env == null) ? "null" : env));
                FileSystem fs = HadoopFileUtils.getFileSystem();
                log.fine("  Default hadoop filesystem: "
                        + ((fs == null) ? "null" : fs.getClass().getSimpleName()));

                if (fs != null) {
                    String dir = MrGeoProperties.getInstance().getProperty(MrGeoConstants.MRGEO_HDFS_IMAGE,
                            "/mrgeo/images");
                    try {
                        log.fine("  Files in " + dir + ":");
                        fs = HadoopFileUtils.getFileSystem(new Path(dir));
                        log.fine("      (" + fs.getClass().getSimpleName() + ")");

                        FileStatus[] files = fs.listStatus(new Path(dir));
                        if (files == null) {
                            log.fine("    <none>");
                        } else {
                            for (FileStatus file : files) {
                                log.fine("    " + file.toString());
                            }
                        }
                    } catch (Exception e) {
                        log.severe("Exception while getting files:");
                        log.severe(e.getMessage());
                        logstack(e);
                        e.printStackTrace();
                    } catch (Throwable t) {
                        log.severe("Throwable (exception) while getting files:");
                        log.severe(t.toString());
                        logstack(t);
                        t.printStackTrace();
                    }
                }
            } catch (Exception e) {
                log.severe("Exception while getting environment:");
                log.severe(e.getMessage());

                logstack(e);
                e.printStackTrace();
            } catch (Throwable t) {
                log.severe("Throwable (exception) while getting environment:");
                log.severe(t.toString());
                logstack(t);
                t.printStackTrace();
            }

        }

        GeoServerResourceLoader loader = (GeoServerResourceLoader) GeoServerExtensions.bean("resourceLoader");
        File file = loader.find(CONFIG_FILE);

        if (file != null && file.exists()) {
            InputStream in = new FileInputStream(file);
            config.load(in);

            config.setProperty(CONFIG_FILE, file.getCanonicalPath());

            if (updater == null) {
                updater = new MrGeoLayerUpdater(config);
            }
        } else {
            log.warning("Can't find the MrGeo config file: " + CONFIG_FILE);
        }
    } catch (Exception e) {
        log.severe("Exception in constructor:");
        log.severe(e.getMessage());
        logstack(e);

        e.printStackTrace();
    } catch (Throwable t) {
        log.severe("Throwable (exception) in constructor:");
        log.severe(t.getMessage());
        logstack(t);

        t.printStackTrace();
    } finally {
        log.fine("Made it to the end of the constructor (" + this.getClass().getSimpleName() + ")");
    }
}