Example usage for org.apache.hadoop.fs AbstractFileSystem createFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs AbstractFileSystem createFileSystem.

Prototype

public static AbstractFileSystem createFileSystem(URI uri, Configuration conf)
        throws UnsupportedFileSystemException 

Source Link

Document

Create a file system instance for the specified uri using the conf.

Usage

From source file:com.cloudera.impala.catalog.TestLoadHdfsMetadataPerf.java

License:Apache License

/**
 * List file status by calling abstractFileSystem.listStatusIterator.
 *//*  w ww.  ja v a2  s . c om*/
private static void listStatusIterator(String dirPath) {
    Path path = new Path(dirPath);
    boolean exceptionThrown = false;
    try {
        AbstractFileSystem fs = AbstractFileSystem.createFileSystem(path.toUri(), LoadMetadataUtil.getConf());
        RemoteIterator<FileStatus> iter = fs.listStatusIterator(path);
        while (iter.hasNext()) {
            FileStatus fileStatus = iter.next();
            BlockLocation[] locations = fs.getFileBlockLocations(fileStatus.getPath(), 0, fileStatus.getLen());
            for (BlockLocation loc : locations) {
                loc.getNames();
                loc.getHosts();
            }
        }
    } catch (IOException e) {
        exceptionThrown = true;
        LOG.error("Failed to list Status Iterator", e);
    }
    assertFalse(exceptionThrown);
}

---

From source file:com.cloudera.impala.util.LoadMetadataUtil.java

License:Apache License

/**
 * Identical to loadFileDescriptors, except using the ListStatusIterator HDFS API to
 * load file status.//w  w  w.j av  a 2  s .  c o  m
 */
public static List<FileDescriptor> loadViaListStatusIterator(FileSystem fs, Path partDirPath,
        Map<String, List<FileDescriptor>> oldFileDescMap, HdfsFileFormat fileFormat,
        Map<FsKey, FileBlocksInfo> perFsFileBlocks, boolean isMarkedCached, String tblName,
        ListMap<TNetworkAddress> hostIndex, Map<String, List<FileDescriptor>> fileDescMap)
        throws FileNotFoundException, IOException {
    List<FileDescriptor> fileDescriptors = Lists.newArrayList();

    AbstractFileSystem abstractFs = AbstractFileSystem.createFileSystem(partDirPath.toUri(), CONF);
    RemoteIterator<FileStatus> fileStatusItor = abstractFs.listStatusIterator(partDirPath);

    while (fileStatusItor.hasNext()) {
        FileStatus fileStatus = fileStatusItor.next();
        FileDescriptor fd = getFileDescriptor(fs, fileStatus, fileFormat, oldFileDescMap, isMarkedCached,
                perFsFileBlocks, tblName, hostIndex);

        if (fd == null)
            continue;

        // Add partition dir to fileDescMap if it does not exist.
        String partitionDir = fileStatus.getPath().getParent().toString();
        if (!fileDescMap.containsKey(partitionDir)) {
            fileDescMap.put(partitionDir, new ArrayList<FileDescriptor>());
        }
        fileDescMap.get(partitionDir).add(fd);

        // Add to the list of FileDescriptors for this partition.
        fileDescriptors.add(fd);
    }

    return fileDescriptors;
}

---

From source file:com.ikanow.aleph2.storage_service_hdfs.services.HdfsStorageService.java

License:Apache License

/** Internal version of getUnderlyingPlatform driver, ie input to cache
 * @param driver_class/*  w  w w .  ja v a2s  .co m*/
 * @param driver_options
 * @return
 */
@SuppressWarnings("unchecked")
public <T> Optional<T> getUnderlyingPlatformDriver_internal(Class<T> driver_class,
        Optional<String> driver_options) {
    T driver = null;
    try {
        if (driver_class != null) {
            final Configuration config = getConfiguration();
            URI uri = driver_options.isPresent() ? new URI(driver_options.get()) : getUri(config);

            if (driver_class.isAssignableFrom(AbstractFileSystem.class)) {

                AbstractFileSystem fs = AbstractFileSystem.createFileSystem(uri, config);

                return (Optional<T>) Optional.of(fs);
            } else if (driver_class.isAssignableFrom(FileContext.class)) {
                FileContext fs = FileContext.getFileContext(AbstractFileSystem.createFileSystem(uri, config),
                        config);
                return (Optional<T>) Optional.of(fs);
            } else if (driver_class.isAssignableFrom(RawLocalFileSystem.class)) {
                return Optional.of(driver_class.newInstance());
            }
        } // !=null
    } catch (Exception e) {
        _logger.error("Caught Exception:", e);
    }
    return Optional.ofNullable(driver);
}

---