Example usage for org.apache.hadoop.fs FileSystem listLocatedStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listLocatedStatus.

Prototype

public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
        throws FileNotFoundException, IOException 

Source Link

Document

List the statuses of the files/directories in the given path if the path is a directory.

Usage

From source file:org.apache.carbondata.hadoop.CarbonInputFormat.java

License:Apache License

private void getFileStatusOfSegments(JobContext job, String[] segmentsToConsider, List<FileStatus> result)
        throws IOException {
    String[] partitionsToConsider = getValidPartitions(job);
    if (partitionsToConsider.length == 0) {
        throw new IOException("No partitions/data found");
    }/*from   w  ww .ja v  a  2s. co  m*/

    PathFilter inputFilter = getDataFileFilter(job);
    CarbonTablePath tablePath = getTablePath(job.getConfiguration());

    // get tokens for all the required FileSystem for table path
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { tablePath }, job.getConfiguration());

    //get all data files of valid partitions and segments
    for (int i = 0; i < partitionsToConsider.length; ++i) {
        String partition = partitionsToConsider[i];

        for (int j = 0; j < segmentsToConsider.length; ++j) {
            String segmentId = segmentsToConsider[j];
            Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
            FileSystem fs = segmentPath.getFileSystem(job.getConfiguration());

            RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
            while (iter.hasNext()) {
                LocatedFileStatus stat = iter.next();
                if (inputFilter.accept(stat.getPath())) {
                    if (stat.isDirectory()) {
                        addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
                    } else {
                        result.add(stat);
                    }
                }
            }
        }
    }
}

---

From source file:org.apache.drill.exec.sql.TestCTAS.java

License:Apache License

@Test
public void createTableWithCustomUmask() throws Exception {
    test("use dfs.tmp");
    String tableName = "with_custom_permission";
    StorageStrategy storageStrategy = new StorageStrategy("000", false);
    FileSystem fs = getLocalFileSystem();
    try {//from   w ww .  j av  a2  s  .c om
        test("alter session set `%s` = '%s'", ExecConstants.PERSISTENT_TABLE_UMASK, storageStrategy.getUmask());
        test("create table %s as select 'A' from (values(1))", tableName);
        Path tableLocation = new Path(dirTestWatcher.getDfsTestTmpDir().getAbsolutePath(), tableName);
        assertEquals("Directory permission should match", storageStrategy.getFolderPermission(),
                fs.getFileStatus(tableLocation).getPermission());
        assertEquals("File permission should match", storageStrategy.getFilePermission(),
                fs.listLocatedStatus(tableLocation).next().getPermission());
    } finally {
        test("alter session reset `%s`", ExecConstants.PERSISTENT_TABLE_UMASK);
        test("drop table if exists %s", tableName);
    }
}

---

From source file:org.apache.druid.storage.hdfs.tasklog.HdfsTaskLogs.java

License:Apache License

@Override
public void killOlderThan(long timestamp) throws IOException {
    Path taskLogDir = new Path(config.getDirectory());
    FileSystem fs = taskLogDir.getFileSystem(hadoopConfig);
    if (fs.exists(taskLogDir)) {

        if (!fs.isDirectory(taskLogDir)) {
            throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir);
        }//from   w  w  w. j a  v  a  2  s.  com

        RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir);
        while (iter.hasNext()) {
            LocatedFileStatus file = iter.next();
            if (file.getModificationTime() < timestamp) {
                Path p = file.getPath();
                log.info("Deleting hdfs task log [%s].", p.toUri().toString());
                fs.delete(p, true);
            }

            if (Thread.currentThread().isInterrupted()) {
                throw new IOException(
                        new InterruptedException("Thread interrupted. Couldn't delete all tasklogs."));
            }
        }
    }
}

---

From source file:org.opencloudengine.garuda.backend.hdfs.HdfsServiceImpl.java

License:Open Source License

@Override
public HdfsListInfo list(String path, int start, int end, final String filter) throws Exception {
    HdfsListInfo hdfsListInfo = new HdfsListInfo();

    this.indexCheck(start, end);
    this.mustExists(path);

    FileSystem fs = fileSystemFactory.getFileSystem();
    Path fsPath = new Path(path);

    FileStatus fileStatus = fs.getFileStatus(fsPath);
    if (!fileStatus.isDirectory()) {
        this.notDirectoryException(fsPath.toString());
    }//w  w  w  .  j a v a2s. c  o  m

    List<HdfsFileInfo> listStatus = new ArrayList<>();
    int count = 0;
    FileStatus fileStatuses = null;
    LocatedFileStatus next = null;
    RemoteIterator<LocatedFileStatus> remoteIterator = fs.listLocatedStatus(fsPath);
    while (remoteIterator.hasNext()) {
        next = remoteIterator.next();
        if (!StringUtils.isEmpty(filter)) {
            if (next.getPath().getName().contains(filter)) {
                count++;
                if (count >= start && count <= end) {
                    fileStatuses = fs.getFileStatus(next.getPath());
                    listStatus
                            .add(new HdfsFileInfo(fileStatuses, fs.getContentSummary(fileStatuses.getPath())));
                }
            }
        } else {
            count++;
            if (count >= start && count <= end) {
                fileStatuses = fs.getFileStatus(next.getPath());
                listStatus.add(new HdfsFileInfo(fileStatuses, fs.getContentSummary(fileStatuses.getPath())));
            }
        }
    }

    hdfsListInfo.setFileInfoList(listStatus);
    hdfsListInfo.setCount(count);
    return hdfsListInfo;
}

---

From source file:uk.ac.ucl.panda.indexing.io.BasicDocMaker.java

License:Apache License

protected void collectFiles(String path, ArrayList inputFiles) throws IOException {

    Path p = new Path(path);
    FileSystem fs = FileSystem.get(new Configuration());
    //System.out.println("Collect: "+f.getAbsolutePath());
    if (!fs.exists(p)) {
        return;//ww  w .  java  2  s. c  o  m
    }
    if (fs.isDirectory(p)) {
        RemoteIterator<LocatedFileStatus> fileIter = fs.listLocatedStatus(p);
        List<String> files = new ArrayList<String>();
        while (fileIter.hasNext()) {
            files.add(fileIter.next().getPath().toString());
        }
        Collections.sort(files);
        for (String f : files) {
            collectFiles(f, inputFiles);
        }
        return;
    }
    //////////////ucl
    if (path.toLowerCase().endsWith("z")) {
        inputFiles.add(path);
        addUniqueBytes(fs.getFileStatus(p).getLen());
    }
}

---

• «
• 1
• 2
• »