Example usage for org.apache.hadoop.fs FileStatus isDirectory

List of usage examples for org.apache.hadoop.fs FileStatus isDirectory

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory()

Source Link

Document

Is this a directory?

Usage

From source file:org.exem.flamingo.shared.util.HdfsUtils.java

License:Apache License

/**
 *  ?   ?  ??./* w  ww. j av a2s.c om*/
 *
 * @param source ?? 
 * @param target ?? 
 * @param fs     Hadoop FileSystem
 */
public static void move(String source, String target, FileSystem fs) throws Exception {
    Path srcPath = new Path(source);
    Path[] srcs = FileUtil.stat2Paths(fs.globStatus(srcPath), srcPath);
    Path dst = new Path(target);
    if (srcs.length > 1 && !fs.getFileStatus(dst).isDirectory()) {
        throw new ServiceException("When moving multiple files, destination should be a directory.");
    }
    for (int i = 0; i < srcs.length; i++) {
        if (!fs.rename(srcs[i], dst)) {
            FileStatus srcFstatus = null;
            FileStatus dstFstatus = null;
            try {
                srcFstatus = fs.getFileStatus(srcs[i]);
            } catch (FileNotFoundException e) {
                throw new FileNotFoundException(srcs[i] + ": No such file or directory");
            }
            try {
                dstFstatus = fs.getFileStatus(dst);
            } catch (IOException e) {
                // Nothing
            }
            if ((srcFstatus != null) && (dstFstatus != null)) {
                if (srcFstatus.isDirectory() && !dstFstatus.isDirectory()) {
                    throw new ServiceException(
                            "cannot overwrite non directory " + dst + " with directory " + srcs[i]);
                }
            }
            throw new ServiceException("Failed to rename " + srcs[i] + " to " + dst);
        }
    }
}

From source file:org.gbif.occurrence.download.oozie.ArchiveBuilder.java

License:Creative Commons License

/**
 * Creates Map with dataset UUIDs and its record counts.
 *//*  www. ja  va2  s  .co m*/
private Map<UUID, Integer> readDatasetCounts(Path citationSrc) throws IOException {
    // the hive query result is a directory with one or more files - read them all into a uuid set
    Map<UUID, Integer> srcDatasets = Maps.newHashMap(); // map of uuids to occurrence counts
    FileStatus[] citFiles = hdfs.listStatus(citationSrc);
    int invalidUuids = 0;
    Closer closer = Closer.create();
    for (FileStatus fs : citFiles) {
        if (!fs.isDirectory()) {
            BufferedReader citationReader = new BufferedReader(
                    new InputStreamReader(hdfs.open(fs.getPath()), Charsets.UTF_8));
            closer.register(citationReader);
            try {
                String line = citationReader.readLine();
                while (line != null) {
                    if (!Strings.isNullOrEmpty(line)) {
                        // we also catch errors for every dataset so we dont break the loop
                        try {
                            Iterator<String> iter = TAB_SPLITTER.split(line).iterator();
                            // play safe and make sure we got a uuid - even though our api doesnt require it
                            UUID key = UUID.fromString(iter.next());
                            Integer count = Integer.parseInt(iter.next());
                            srcDatasets.put(key, count);
                            // small downloads persist dataset usages while builds the citations file
                            if (!isSmallDownload) {
                                persistDatasetUsage(count, downloadId, key);
                            }
                        } catch (IllegalArgumentException e) {
                            // ignore invalid UUIDs
                            LOG.info("Found invalid UUID as datasetId {}", line);
                            invalidUuids++;
                        }
                    }
                    line = citationReader.readLine();
                }
            } finally {
                closer.close();
            }
        }
    }
    if (invalidUuids > 0) {
        LOG.info("Found {} invalid dataset UUIDs", invalidUuids);
    } else {
        LOG.info("All {} dataset UUIDs are valid", srcDatasets.size());
    }
    return srcDatasets;
}

From source file:org.gridgain.grid.kernal.ggfs.hadoop.GridGgfsHadoopFileSystemWrapper.java

License:Open Source License

/** {@inheritDoc} */
@Override/* ww w .  j  ava 2s  . c o m*/
public Collection<GridGgfsFile> listFiles(GridGgfsPath path) throws GridException {
    try {
        FileStatus[] statuses = fileSys.listStatus(convert(path));

        if (statuses == null)
            throw new GridGgfsFileNotFoundException("Failed to list files (path not found): " + path);

        Collection<GridGgfsFile> res = new ArrayList<>(statuses.length);

        for (FileStatus status : statuses) {
            GridGgfsFileInfo fsInfo = status.isDirectory() ? new GridGgfsFileInfo(true, properties(status))
                    : new GridGgfsFileInfo((int) status.getBlockSize(), status.getLen(), null, null, false,
                            properties(status));

            res.add(new GridGgfsFileImpl(new GridGgfsPath(path, status.getPath().getName()), fsInfo, 1));
        }

        return res;
    } catch (FileNotFoundException ignored) {
        throw new GridGgfsFileNotFoundException("Failed to list files (path not found): " + path);
    } catch (IOException e) {
        throw handleSecondaryFsError(e,
                "Failed to list statuses due to secondary file system exception: " + path);
    }
}

From source file:org.gridgain.grid.kernal.ggfs.hadoop.GridGgfsHadoopFileSystemWrapper.java

License:Open Source License

/** {@inheritDoc} */
@Override//from   www .j a v a 2  s.com
public GridGgfsFile info(final GridGgfsPath path) throws GridException {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new GridGgfsFile() {
            @Override
            public GridGgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                return (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };

    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

From source file:org.hedera.io.input.WikiRevisionInputFormat.java

License:Apache License

/** 
 * This code is copied from StreamWikiDumpNewInputFormat.java by Yusuke Matsubara.
 * Thanks to Tu Meteora for adjusting the code to the new mapreduce framework
 * @param job the job context/*from ww w . j a  v a2s .c  om*/
 * @throws IOException
 */
@Override
public List<InputSplit> getSplits(JobContext jc) throws IOException {

    List<FileStatus> files = listStatus(jc);
    List<FileStatus> remainingFiles = new ArrayList<>();

    List<InputSplit> splits = new ArrayList<InputSplit>();
    long totalSize = 0;

    // New features: Load splits from the index
    // Check the index before performing the split on the physical files
    Configuration conf = jc.getConfiguration();

    String mapFile = conf.get(SPLIT_MAPFILE_LOC);
    MapFile.Reader reader = null;
    Text key = null;
    RevisionSplits val = new RevisionSplits();
    try {
        if (mapFile != null) {
            reader = new MapFile.Reader(new Path(mapFile + "/part-r-00000"), conf);
            key = new Text();
        }

        // check we have valid files
        for (FileStatus file : files) {
            if (file.isDirectory()) {
                throw new IOException("Not a file: " + file.getPath());
            }

            // if found in the index, load the splits into main memory, otherwise
            // add to remainings for next processing
            if (reader != null) {
                key.set(file.getPath().toString());
                if (reader.seek(key)) {
                    reader.get(key, val);
                    FileSplit[] spl = val.splits();
                    for (FileSplit sp : spl)
                        splits.add(sp);
                    continue;
                }
            }
            remainingFiles.add(file);
            totalSize += file.getLen();
        }
        long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(jc));

        // 2014-06-06: Tuan _ I have to manually increase the file split size
        // here to cope with Wikipedia Revision .bz2 file - the decompressor
        // takes too long to run
        long goalSize = totalSize / 3;

        for (FileStatus file : remainingFiles) {
            long blockSize = file.getBlockSize();
            long splitSize = computeSplitSize(goalSize, minSize, blockSize);

            for (InputSplit x : getSplits(jc, file, splitSize))
                splits.add(x);
        }
    } finally {
        if (reader != null)
            reader.close();
    }

    return splits;
}

From source file:org.lafs.hdfs.LAFS.java

License:Apache License

@Override
public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException {

    long offset = 0;
    boolean isDir = false;

    // if the file exists, find file size to use as offset in PUT call
    try {// w w w. j a v  a2s .  co  m
        FileStatus status = getFileStatus(path);

        isDir = status.isDirectory();

        offset = status.getLen();
    } catch (IOException ioe) {
        // file doesn't exist
    }

    if (isDir)
        throw new IOException("Cannot append. Path is a directory");

    String req = httpURI + "/uri/" + getLAFSPath(path);
    req += "?format=MDMF&offset=" + offset;
    URL url = new URL(req);
    //System.out.println(req);

    // Open the connection and prepare to POST
    HttpURLConnection uc = (HttpURLConnection) url.openConnection();
    uc.setDoOutput(true);
    uc.setRequestMethod("PUT");
    uc.setChunkedStreamingMode(writeChunkSize);

    return new FSDataOutputStream(new LAFSOutputStream(uc)); //, this, path));
}

From source file:org.mule.modules.hdfs.automation.functional.legacy.GetMetadataTestCases.java

License:Open Source License

@Test
public void testGetMetadata() throws Exception {
    MuleMessage muleMessage = runFlowAndGetMessage("get-metadata");

    boolean exists = (Boolean) muleMessage.getInvocationProperty(HDFSConnector.HDFS_PATH_EXISTS);
    assertTrue(exists);/* w w w.ja  v a  2 s . c o  m*/

    MD5MD5CRC32FileChecksum fileMd5 = muleMessage.getInvocationProperty(HDFSConnector.HDFS_FILE_CHECKSUM);
    assertNotNull(fileMd5);

    FileStatus fileStatus = muleMessage.getInvocationProperty(HDFSConnector.HDFS_FILE_STATUS);
    assertFalse(fileStatus.isDirectory());

    ContentSummary contentSummary = muleMessage.getInvocationProperty(HDFSConnector.HDFS_CONTENT_SUMMARY);
    assertNotNull(contentSummary);
}

From source file:org.mule.modules.hdfs.automation.testcases.GetMetadataTestCases.java

License:Open Source License

@Category({ SmokeTests.class, RegressionTests.class })
@Test//from  w ww  .  j a v a2s  . co m
public void testGetMetadata() {
    try {
        MuleMessage muleMessage = runFlowAndGetMessage("get-metadata");

        boolean exists = (Boolean) muleMessage.getInvocationProperty(HDFSConnector.HDFS_PATH_EXISTS);
        assertTrue(exists);

        MD5MD5CRC32FileChecksum fileMd5 = muleMessage.getInvocationProperty(HDFSConnector.HDFS_FILE_CHECKSUM);
        assertNotNull(fileMd5);

        FileStatus fileStatus = muleMessage.getInvocationProperty(HDFSConnector.HDFS_FILE_STATUS);
        assertFalse(fileStatus.isDirectory());

        ContentSummary contentSummary = muleMessage.getInvocationProperty(HDFSConnector.HDFS_CONTENT_SUMMARY);
        assertNotNull(contentSummary);

    } catch (Exception e) {
        fail(ConnectorTestUtils.getStackTrace(e));
    }
}

From source file:org.mule.modules.hdfs.HDFSConnector.java

License:Open Source License

private Map<String, Object> getPathMetaData(final Path hdfsPath) throws IOException {
    final Map<String, Object> metaData = new HashMap<String, Object>();

    final boolean pathExists = fileSystem.exists(hdfsPath);
    metaData.put(HDFS_PATH_EXISTS, pathExists);
    if (!pathExists) {
        return metaData;
    }//w ww.  java  2  s.c o m

    metaData.put(HDFS_CONTENT_SUMMARY, fileSystem.getContentSummary(hdfsPath));

    final FileStatus fileStatus = fileSystem.getFileStatus(hdfsPath);
    metaData.put(HDFS_FILE_STATUS, fileStatus);
    if (fileStatus.isDirectory()) {
        return metaData;
    }

    final FileChecksum fileChecksum = fileSystem.getFileChecksum(hdfsPath);
    if (fileChecksum != null) {
        metaData.put(HDFS_FILE_CHECKSUM, fileChecksum);
    }

    return metaData;
}

From source file:org.nchadoop.fs.HdfsScanner.java

License:Apache License

private void walkThroughDirectories(final StatusCallback callback, final SearchRoot searchRoot,
        final FileStatus[] listLocatedStatus) throws FileNotFoundException, IOException {
    for (final FileStatus fileStatus : listLocatedStatus) {
        if (fileStatus.isDirectory()) {
            try {
                walkThroughDirectories(callback, searchRoot, this.fileSystem.listStatus(fileStatus.getPath()));
            } catch (final IOException e) {
                log.warn("Couldn't open directory {}. Exception: {}", fileStatus.getPath(), e.getMessage());
            }//from w w  w  .j av a 2  s. c o  m
        } else {
            if (callback != null) {
                callback.onVisitFile(fileStatus);
            }
            addFile(searchRoot, fileStatus);
        }
    }

}