Example usage for org.apache.hadoop.fs FileStatus isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory() 

Source Link

Document

Is this a directory?

Usage

From source file:com.ikanow.aleph2.core.shared.utils.DirUtils.java

License:Apache License

/** This method returns the path to the first subdirectory matching the subDirectoryName parameter or null if not found.
* @param fileContext//from   www  .j a  va2s  .c  o  m
* @param start
* @param subDirectoryName
* @return
*/
public static Path findOneSubdirectory(FileContext fileContext, Path start, String subDirectoryName) {
    Path p = null;
    try {
        logger.debug("findOneSubdirectory :" + start.toString());
        FileStatus[] statuss = fileContext.util().listStatus(start);
        for (int i = 0; i < statuss.length; i++) {
            FileStatus dir = statuss[i];
            logger.debug("FileStatus:" + statuss[i].getPath().toString());
            if (dir.isDirectory()) {
                if (dir.getPath().getName().contains(subDirectoryName)) {
                    logger.debug("findOneSubdirectory match:" + dir.getPath().getName());
                    return dir.getPath();
                } else {
                    p = findOneSubdirectory(fileContext, dir.getPath(), subDirectoryName);
                    if (p != null) {
                        return p;
                    }
                }
            }
        }

    } catch (Exception e) {
        logger.error("findOneSubdirectory Caught Exception", e);
    }

    return p;
}

---

From source file:com.ikanow.aleph2.core.shared.utils.DirUtils.java

License:Apache License

/**
 * @param allPaths/*from  w  w  w.  j a  va  2s.  c  o m*/
 * @param fileContext
 * @param start
 * @param subDirectoryName
 * @param includeMatched
 */
public static void findAllSubdirectories(List<Path> allPaths, FileContext fileContext, Path start,
        String subDirectoryName, boolean includeMatched) {
    try {
        logger.debug("findAllSubdirectories :" + start.toString());
        FileStatus[] statuss = fileContext.util().listStatus(start);
        for (int i = 0; i < statuss.length; i++) {
            FileStatus dir = statuss[i];
            logger.debug("FileStatus:" + statuss[i].getPath().toString());
            if (dir.isDirectory()) {
                if (dir.getPath().getName().contains(subDirectoryName)) {
                    logger.debug("findOneSubdirectory match:" + dir.getPath().getName());
                    if (includeMatched) {
                        allPaths.add(dir.getPath());
                    } else {
                        allPaths.add(dir.getPath().getParent());
                    }
                } else {
                    findAllSubdirectories(allPaths, fileContext, dir.getPath(), subDirectoryName,
                            includeMatched);
                }
            }
        }

    } catch (Exception e) {
        logger.error("findAllSubdirectories Caught Exception", e);
    }
}

---

From source file:com.indeed.iupload.core.filesystem.HDFSProxy.java

License:Apache License

@Override
public List<String> listFiles(String path) throws IOException {
    final Path pathObj = new Path(path);
    if (!getFileSystem(path).exists(pathObj)) {
        return Lists.newArrayList();
    }//  w  w w  .j a  va  2 s  .  c  o m
    List<String> results = new ArrayList<String>();
    for (FileStatus status : getFileSystem(path).listStatus(pathObj)) {
        if (!status.isDirectory()) {
            results.add(status.getPath().getName());
        }
    }
    return results;
}

---

From source file:com.indeed.iupload.core.filesystem.HDFSProxy.java

License:Apache License

@Override
public List<String> listDirs(String path) throws IOException {
    final Path pathObj = new Path(path);
    if (!getFileSystem(path).exists(pathObj)) {
        return Lists.newArrayList();
    }/*from  w  ww. j a va  2s.  c  o m*/
    List<String> results = new ArrayList<String>();
    for (FileStatus status : getFileSystem(path).listStatus(pathObj)) {
        if (status.isDirectory()) {
            results.add(status.getPath().getName());
        }
    }
    return results;
}

---

From source file:com.intropro.prairie.unit.hdfs.HdfsUnit.java

License:Apache License

public <T> CompareResponse<T> compare(Path path, Format<T> format, InputStream expectedStream,
        Format<T> expectedFormat) throws IOException {
    InputStream inputStream = null;
    if (getFileSystem().isDirectory(path)) {
        for (FileStatus fileStatus : getFileSystem().listStatus(path)) {
            if (fileStatus.isDirectory()) {
                throw new IOException(fileStatus.getPath().toString() + " is directory");
            }/*from w ww.  j  a va2s . c  o  m*/
            InputStream fileStream = getFileSystem().open(fileStatus.getPath());
            if (inputStream == null) {
                inputStream = fileStream;
            } else {
                inputStream = new SequenceInputStream(inputStream, fileStream);
            }
        }
    } else {
        inputStream = getFileSystem().open(path);
    }
    try {
        InputFormatReader<T> reader = format.createReader(inputStream);
        InputFormatReader<T> expectedReader = expectedFormat.createReader(expectedStream);
        CompareResponse<T> compareResponse = byLineComparator.compare(expectedReader.all(), reader.all());
        expectedReader.close();
        expectedReader.close();
        return compareResponse;
    } catch (FormatException e) {
        throw new IOException(e);
    }
}

---

From source file:com.jaeksoft.searchlib.crawler.cache.HadoopCrawlCache.java

License:Open Source License

private final long purge(FileStatus[] files, long expiration) throws IOException {
    long count = 0;
    for (FileStatus file : files) {
        if (file.isDirectory()) {
            Path p = file.getPath();
            count += purge(fileSystem.listStatus(p), expiration);
            FileStatus[] fs = fileSystem.listStatus(p);
            if (fs.length == 0)
                if (fileSystem.delete(p, false))
                    count++;/*from   w  w  w.j a v a 2  s  .c o  m*/
        } else {
            if (file.getModificationTime() < expiration)
                if (fileSystem.delete(file.getPath(), false))
                    count++;
        }
    }
    return count;
}

---

From source file:com.jeffy.hdfs.FileMetaData.java

License:Apache License

public static void showFileStatusForFile(String path) {
    Configuration config = new Configuration();
    try {//  www  .  j  av  a2 s .co  m
        FileSystem fs = FileSystem.get(URI.create(path), config);
        FileStatus stat = fs.getFileStatus(new Path(path));
        System.out.println("File URI: " + stat.getPath().toUri().getPath());
        System.out.println("Is directory: " + stat.isDirectory());
        System.out.println("File length: " + stat.getLen());
        System.out.println("Modification Time: " + new Date(stat.getModificationTime()));
        System.out.println("File replications: " + stat.getReplication());
        System.out.println("File Block Size: " + (stat.getBlockSize() >>> 10 >>> 10) + " MB");
        System.out.println("File Owner: " + stat.getOwner());
        System.out.println("File Group: " + stat.getGroup());
        System.out.println("File Permission: " + stat.getPermission().toString());
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:com.knewton.mapreduce.io.SSTableInputFormat.java

License:Apache License

/**
 * Expands all directories passed as input and keeps only valid data tables.
 *
 * @return A list of all the data tables found under the input directories.
 *//*from w w  w .j av a2  s. com*/
@Override
protected List<FileStatus> listStatus(JobContext job) throws IOException {
    Configuration conf = job.getConfiguration();
    List<FileStatus> files = super.listStatus(job);
    DataTablePathFilter dataTableFilter = getDataTableFilter(conf);
    files = cleanUpBackupDir(files);
    for (int i = 0; i < files.size(); i++) {
        FileStatus file = files.get(i);
        Path p = file.getPath();
        // Expand if directory
        if (file.isDirectory() && p != null) {
            LOG.info("Expanding {}", p);
            FileSystem fs = p.getFileSystem(conf);
            FileStatus[] children = fs.listStatus(p);
            List<FileStatus> listChildren = Lists.newArrayList(children);
            listChildren = cleanUpBackupDir(listChildren);
            files.addAll(i + 1, listChildren);
        }
        if (!dataTableFilter.accept(file.getPath())) {
            LOG.info("Removing {}", file.getPath());
            files.remove(i);
            i--;
        }
    }
    return files;
}

---

From source file:com.knewton.mapreduce.SSTableRecordReader.java

License:Apache License

/**
 * Copies a remote path to the local filesystem, while updating hadoop that we're making
 * progress. Doesn't support directories.
 *//*  ww w .  j a v  a 2  s.co  m*/
@VisibleForTesting
void copyToLocalFile(FileSystem remoteFS, FileSystem localFS, Path remote, Path local) throws IOException {
    // don't support transferring from remote directories
    FileStatus remoteStat = remoteFS.getFileStatus(remote);
    Preconditions.checkArgument(!remoteStat.isDirectory(), String.format("Path %s is directory!", remote));
    // if local is a dir, copy to inside that dir, like 'cp /path/file /tmp/' would do
    if (localFS.exists(local)) {
        FileStatus localStat = localFS.getFileStatus(local);
        if (localStat.isDirectory()) {
            local = new Path(local, remote.getName());
        }
    }
    long remoteFileSize = remoteStat.getLen();
    // do actual copy
    InputStream in = null;
    OutputStream out = null;
    try {
        long startTime = System.currentTimeMillis();
        long lastLogTime = 0;
        long bytesCopied = 0;
        in = remoteFS.open(remote);
        out = localFS.create(local, true);
        int buffSize = this.conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY,
                CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_DEFAULT);
        byte[] buf = new byte[buffSize];
        int bytesRead = in.read(buf);
        while (bytesRead >= 0) {
            long now = System.currentTimeMillis();
            // log transfer rate once per min, starting 1 min after transfer began
            if (now - lastLogTime > 60000L && now - startTime > 60000L) {
                double elapsedSec = (now - startTime) / 1000D;
                double bytesPerSec = bytesCopied / elapsedSec;
                LOG.info("Transferred {} of {} bytes at {} bytes per second", bytesCopied, remoteFileSize,
                        bytesPerSec);
                lastLogTime = now;
            }
            this.ctx.progress();
            out.write(buf, 0, bytesRead);
            bytesCopied += bytesRead;
            bytesRead = in.read(buf);
        }
        // try to close these outside of finally so we receive exception on failure
        out.close();
        out = null;
        in.close();
        in = null;
    } finally {
        // make sure everything's closed
        IOUtils.closeStream(out);
        IOUtils.closeStream(in);
    }
}

---

From source file:com.knewton.mapreduce.SSTableRecordReaderTest.java

License:Apache License

/**
 * Tests to see if tables can be correctly copied locally
 *//*from   w  w  w  .  j a va 2s . c om*/
@Test
public void testCopyTablesToLocal() throws Exception {
    TaskAttemptContext context = getTaskAttemptContext(true, true, true);
    ssTableColumnRecordReader.initialize(inputSplit, context);

    doCallRealMethod().when(ssTableColumnRecordReader).copyTablesToLocal(any(FileSystem.class),
            any(FileSystem.class), any(Path.class), any(TaskAttemptContext.class));

    FileSystem remoteFS = mock(FileSystem.class);
    FileSystem localFS = mock(FileSystem.class);

    byte[] data = new byte[] { 0xA };
    FSDataInputStream fsIn = new FSDataInputStream(new MemoryDataInputStream(data));
    FSDataOutputStream fsOut = mock(FSDataOutputStream.class);

    when(remoteFS.open(any(Path.class))).thenReturn(fsIn);
    when(localFS.create(any(Path.class), anyBoolean())).thenReturn(fsOut);

    Path dataTablePath = inputSplit.getPath();
    FileStatus fileStatus = mock(FileStatus.class);
    when(fileStatus.getLen()).thenReturn(10L);
    when(fileStatus.isDirectory()).thenReturn(false);
    when(remoteFS.getFileStatus(any(Path.class))).thenReturn(fileStatus);

    ssTableColumnRecordReader.copyTablesToLocal(remoteFS, localFS, dataTablePath, context);
    verify(remoteFS).getFileStatus(dataTablePath);
    ssTableColumnRecordReader.close();
    verify(fsOut).write(any(byte[].class), eq(0), eq(data.length));
    assertEquals(2, ssTableColumnRecordReader.getComponentSize());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »