Example usage for org.apache.hadoop.fs FileStatus isDirectory

List of usage examples for org.apache.hadoop.fs FileStatus isDirectory

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory()

Source Link

Document

Is this a directory?

Usage

From source file:com.datatorrent.stram.client.RecordingsAgent.java

License:Apache License

private RecordingInfo getRecordingInfoHelper(String appId, String opId, String id, Set<String> containers) {
    RecordingInfo info = new RecordingInfo();
    info.id = id;// w w  w.jav  a2 s  .  c  om
    info.appId = appId;
    info.operatorId = opId;

    BufferedReader br = null;
    IndexFileBufferedReader ifbr = null;
    try {
        String dir = getRecordingDirectory(appId, opId, id);
        if (dir == null) {
            throw new Exception("recording directory is null");
        }

        Path path = new Path(dir);
        JSONObject json;

        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        HashMap<String, PortInfo> portMap = new HashMap<String, PortInfo>();
        if (!fileStatus.isDirectory()) {
            throw new Exception(path + " is not a directory");
        }

        // META file processing
        br = new BufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.META_FILE))));
        String line;
        line = br.readLine();
        if (!line.equals("1.2")) {
            throw new Exception("Unexpected line: " + line);
        }
        line = br.readLine();
        json = new JSONObject(line);
        info.startTime = json.getLong("startTime");
        info.containerId = json.optString("containerId");
        info.properties = new HashMap<String, Object>();

        if (!StringUtils.isBlank(info.containerId) && !containers.contains(info.containerId)) {
            info.ended = true;
        }

        json = json.optJSONObject("properties");
        if (json != null) {
            @SuppressWarnings("unchecked")
            Iterator<String> keys = json.keys();
            while (keys.hasNext()) {
                String key = keys.next();
                // ugly 2 lines of code below since JSONObject.get(key).toString() doesn't give you json representation for plain strings
                String strValue = json.isNull(key) ? null : json.optString(key);
                info.properties.put(key,
                        strValue != null ? strValue : new ObjectMapperString(json.get(key).toString()));
            }
        }
        info.ports = new ArrayList<PortInfo>();
        while ((line = br.readLine()) != null) {
            PortInfo portInfo = new PortInfo();
            json = new JSONObject(line);
            portInfo.id = json.getInt("id");
            portInfo.name = json.getString("name");
            portInfo.type = json.getString("type");
            portInfo.streamName = json.getString("streamName");
            info.ports.add(portInfo);
            portMap.put(String.valueOf(portInfo.id), portInfo);
        }

        // INDEX file processing
        ifbr = new IndexFileBufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.INDEX_FILE))), dir);
        info.windowIdRanges = new ArrayList<TupleRecorder.Range>();
        long prevHiWindowId = -1;
        RecordingsIndexLine indexLine;
        while ((indexLine = (RecordingsIndexLine) ifbr.readIndexLine()) != null) {
            if (indexLine.isEndLine) {
                info.ended = true;
            } else {
                info.totalTuples += indexLine.tupleCount;
                for (Map.Entry<String, MutableLong> entry : indexLine.portTupleCount.entrySet()) {
                    PortInfo portInfo = portMap.get(entry.getKey());
                    if (portInfo == null) {
                        throw new Exception("port info does not exist for " + entry.getKey());
                    }
                    portInfo.tupleCount += entry.getValue().longValue();
                }
                for (TupleRecorder.Range r : indexLine.windowIdRanges) {
                    if (info.windowIdRanges.isEmpty()) {
                        TupleRecorder.Range range = new TupleRecorder.Range();
                        range.low = r.low;
                        info.windowIdRanges.add(range);
                    } else if (prevHiWindowId + 1 != r.low) {
                        TupleRecorder.Range range = info.windowIdRanges.get(info.windowIdRanges.size() - 1);
                        range.high = prevHiWindowId;
                        range = new TupleRecorder.Range();
                        range.low = r.low;
                        info.windowIdRanges.add(range);
                    }
                    prevHiWindowId = r.high;
                }
            }
        }
        if (!info.windowIdRanges.isEmpty()) {
            TupleRecorder.Range range = info.windowIdRanges.get(info.windowIdRanges.size() - 1);
            range.high = prevHiWindowId;
        }
    } catch (Exception ex) {
        LOG.warn("Got exception when getting recording info", ex);
        return null;
    } finally {
        IOUtils.closeQuietly(ifbr);
        IOUtils.closeQuietly(br);
    }

    return info;
}

From source file:com.datatorrent.stram.client.StatsAgent.java

License:Apache License

public ContainersInfo getContainersInfo(String appId) {
    ContainersInfo info = new ContainersInfo();
    info.appId = appId;//from  ww  w  . java  2  s .  co  m
    info.containers = new HashMap<Integer, ContainerInfo>();
    String dir = getContainerStatsDirectory(appId);
    if (dir == null) {
        return null;
    }
    Path path = new Path(dir);
    JSONObject json;
    BufferedReader br = null;
    IndexFileBufferedReader ifbr = null;
    try {
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        if (!fileStatus.isDirectory()) {
            return null;
        }

        // META file processing
        br = new BufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.META_FILE))));
        String line;
        line = br.readLine();
        if (!line.equals("1.0")) {
            return null;
        }
        while ((line = br.readLine()) != null) {
            int cursor = line.indexOf(':');
            int index = Integer.valueOf(line.substring(0, cursor));
            json = new JSONObject(line.substring(cursor + 1));
            ContainerInfo containerInfo = new ContainerInfo();
            containerInfo.host = json.getString("host");
            containerInfo.jvmName = json.getString("jvmName");
            containerInfo.id = json.getString("id");
            containerInfo.memoryMBAllocated = json.getInt("memoryMBAllocated");
            info.containers.put(index, containerInfo);
        }
        // INDEX file processing
        ifbr = new IndexFileBufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.INDEX_FILE))), dir);
        StatsIndexLine indexLine;

        while ((indexLine = (StatsIndexLine) ifbr.readIndexLine()) != null) {
            if (indexLine.isEndLine) {
                info.ended = true;
            } else {
                info.count += indexLine.count;
                if (info.startTime == 0 || info.startTime > indexLine.startTime) {
                    info.startTime = indexLine.startTime;
                }
                if (info.endTime == 0 || info.endTime < indexLine.endTime) {
                    info.endTime = indexLine.endTime;
                }
            }
        }
    } catch (Exception ex) {
        LOG.warn("Got exception when reading containers info", ex);
        return null;
    } finally {
        IOUtils.closeQuietly(br);
        IOUtils.closeQuietly(ifbr);
    }

    return info;
}

From source file:com.datatorrent.stram.client.StatsAgent.java

License:Apache License

public OperatorsInfo getOperatorsInfo(String appId, String opName) {
    OperatorsInfo info = new OperatorsInfo();
    info.appId = appId;//from  ww  w .j  a v a 2 s.c  o m
    info.operatorName = opName;
    info.operatorIds = new ArrayList<Integer>();
    String dir = getOperatorStatsDirectory(appId, opName);
    if (dir == null) {
        return null;
    }

    Path path = new Path(dir);
    JSONObject json;
    BufferedReader br = null;
    IndexFileBufferedReader ifbr = null;

    try {
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        if (!fileStatus.isDirectory()) {
            return null;
        }

        // META file processing
        br = new BufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.META_FILE))));
        String line;
        line = br.readLine();
        if (!line.equals("1.0")) {
            return null;
        }
        while ((line = br.readLine()) != null) {
            json = new JSONObject(line);
            info.operatorIds.add(json.getInt("id"));
        }

        // INDEX file processing
        ifbr = new IndexFileBufferedReader(new InputStreamReader(
                stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.INDEX_FILE))), dir);
        StatsIndexLine indexLine;
        while ((indexLine = (StatsIndexLine) ifbr.readIndexLine()) != null) {
            if (indexLine.isEndLine) {
                info.ended = true;
            } else {
                info.count += indexLine.count;
                if (info.startTime == 0 || info.startTime > indexLine.startTime) {
                    info.startTime = indexLine.startTime;
                }
                if (info.endTime == 0 || info.endTime < indexLine.endTime) {
                    info.endTime = indexLine.endTime;
                }
            }
        }
    } catch (Exception ex) {
        LOG.warn("Got exception when reading operators info", ex);
        return null;
    } finally {
        IOUtils.closeQuietly(ifbr);
        IOUtils.closeQuietly(br);
    }

    return info;
}

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

public void copyInitialState(Path origAppDir) throws IOException {
    // locate previous snapshot
    String newAppDir = this.dag.assertAppPath();

    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
    // read snapshot against new dependencies
    Object snapshot = recoveryHandler.restore();
    if (snapshot == null) {
        throw new IllegalArgumentException("No previous application state found in " + origAppDir);
    }/*from  ww  w . j  a  va2s .  co m*/
    InputStream logIs = recoveryHandler.getLog();

    // modify snapshot state to switch app id
    ((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
    Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);

    FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
    // remove the path that was created by the storage agent during deserialization and replacement
    fs.delete(checkpointPath, true);

    // write snapshot to new location
    recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
    recoveryHandler.save(snapshot);
    OutputStream logOs = recoveryHandler.rotateLog();
    IOUtils.copy(logIs, logOs);
    logOs.flush();
    logOs.close();
    logIs.close();

    // copy sub directories that are not present in target
    FileStatus[] lFiles = fs.listStatus(origAppDir);
    for (FileStatus f : lFiles) {
        if (f.isDirectory()) {
            String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
            if (!fs.exists(new Path(targetPath))) {
                LOG.debug("Copying {} to {}", f.getPath(), targetPath);
                FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
                //FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
            } else {
                LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
                //FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
            }
        }
    }

}

From source file:com.datatorrent.stram.util.FSUtil.java

License:Apache License

/**
 * Copied from FileUtil to transfer ownership
 *
 * @param srcFS//from  www . j  a v  a  2s  .co m
 * @param srcStatus
 * @param dstFS
 * @param dst
 * @param deleteSource
 * @param overwrite
 * @param conf
 * @return
 * @throws IOException
 */
public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst,
        boolean deleteSource, boolean overwrite, Configuration conf) throws IOException {
    Path src = srcStatus.getPath();
    //dst = checkDest(src.getName(), dstFS, dst, overwrite);
    if (srcStatus.isDirectory()) {
        //checkDependencies(srcFS, src, dstFS, dst);
        if (!mkdirs(dstFS, dst)) {
            return false;
        }

        FileStatus contents[] = srcFS.listStatus(src);
        for (int i = 0; i < contents.length; i++) {
            copy(srcFS, contents[i], dstFS, new Path(dst, contents[i].getPath().getName()), deleteSource,
                    overwrite, conf);
        }
    } else {
        InputStream in = null;
        OutputStream out = null;
        try {
            in = srcFS.open(src);
            out = dstFS.create(dst, overwrite);
            org.apache.hadoop.io.IOUtils.copyBytes(in, out, conf, true);
        } catch (IOException e) {
            org.apache.hadoop.io.IOUtils.closeStream(out);
            org.apache.hadoop.io.IOUtils.closeStream(in);
            throw e;
        }
    }

    // TODO: change group and limit write to group
    if (srcStatus.isDirectory()) {
        dstFS.setPermission(dst, new FsPermission((short) 0777));
    } else {
        dstFS.setPermission(dst, new FsPermission((short) 0777)/*"ugo+w"*/);
    }
    //dstFS.setOwner(dst, null, srcStatus.getGroup());

    /*
        try {
          // transfer owner
          // DOES NOT WORK only super user can change file owner
          dstFS.setOwner(dst, srcStatus.getOwner(), srcStatus.getGroup());
        } catch (IOException e) {
          LOG.warn("Failed to change owner on {} to {}", dst, srcStatus.getOwner(), e);
          throw e;
        }
    */
    if (deleteSource) {
        return srcFS.delete(src, true);
    } else {
        return true;
    }

}

From source file:com.dinglicom.clouder.mapreduce.input.FileInputFormat.java

License:Apache License

/** List input directories.
 * Subclasses may override to, e.g., select only files matching a regular
 * expression. //from   ww  w . j av  a 2  s .  c  om
 * 
 * @param job the job to list input paths for
 * @return array of FileStatus objects
 * @throws IOException if zero items.
 */
protected List<FileStatus> listStatus(JobContext job) throws IOException {
    List<FileStatus> result = new ArrayList<FileStatus>();
    Path[] dirs = getInputPaths(job);
    if (dirs.length == 0) {
        throw new IOException("No input paths specified in job");
    }

    // get tokens for all the required FileSystems..
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), dirs, job.getConfiguration());

    List<IOException> errors = new ArrayList<IOException>();

    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
        filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);

    for (int i = 0; i < dirs.length; ++i) {
        Path p = dirs[i];
        FileSystem fs = p.getFileSystem(job.getConfiguration());
        FileStatus[] matches = fs.globStatus(p, inputFilter);
        if (matches == null) {
            errors.add(new IOException("Input path does not exist: " + p));
        } else if (matches.length == 0) {
            errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
        } else {
            for (FileStatus globStat : matches) {
                if (globStat.isDirectory()) {
                    for (FileStatus stat : fs.listStatus(globStat.getPath(), inputFilter)) {
                        result.add(stat);
                    }
                } else {
                    result.add(globStat);
                }
            }
        }
    }

    if (!errors.isEmpty()) {
        throw new InvalidInputException(errors);
    }
    LOG.info("Total input paths to process : " + result.size());
    return result;
}

From source file:com.facebook.presto.hadoop.HadoopFileStatus.java

License:Apache License

public static boolean isDirectory(FileStatus status) {
    return status.isDirectory();
}

From source file:com.facebook.presto.hive.metastore.file.FileHiveMetastore.java

License:Apache License

private List<ArrayDeque<String>> listPartitions(Path director, List<Column> partitionColumns) {
    if (partitionColumns.isEmpty()) {
        return ImmutableList.of();
    }/*from  w w  w.  j  a va 2s  . co m*/

    try {
        String directoryPrefix = partitionColumns.get(0).getName() + '=';

        List<ArrayDeque<String>> partitionValues = new ArrayList<>();
        for (FileStatus fileStatus : metadataFileSystem.listStatus(director)) {
            if (!fileStatus.isDirectory()) {
                continue;
            }
            if (!fileStatus.getPath().getName().startsWith(directoryPrefix)) {
                continue;
            }

            List<ArrayDeque<String>> childPartitionValues;
            if (partitionColumns.size() == 1) {
                childPartitionValues = ImmutableList.of(new ArrayDeque<>());
            } else {
                childPartitionValues = listPartitions(fileStatus.getPath(),
                        partitionColumns.subList(1, partitionColumns.size()));
            }

            String value = fileStatus.getPath().getName().substring(directoryPrefix.length());
            for (ArrayDeque<String> childPartition : childPartitionValues) {
                childPartition.addFirst(value);
                partitionValues.add(childPartition);
            }
        }
        return partitionValues;
    } catch (IOException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, "Error listing partition directories", e);
    }
}

From source file:com.facebook.presto.hive.metastore.file.FileHiveMetastore.java

License:Apache License

private List<Path> getChildSchemaDirectories(Path metadataDirectory) {
    try {/*ww  w . j a va2 s. c om*/
        if (!metadataFileSystem.isDirectory(metadataDirectory)) {
            return ImmutableList.of();
        }

        ImmutableList.Builder<Path> childSchemaDirectories = ImmutableList.builder();
        for (FileStatus child : metadataFileSystem.listStatus(metadataDirectory)) {
            if (!child.isDirectory()) {
                continue;
            }
            Path childPath = child.getPath();
            if (childPath.getName().startsWith(".")) {
                continue;
            }
            if (metadataFileSystem.isFile(new Path(childPath, PRESTO_SCHEMA_FILE_NAME))) {
                childSchemaDirectories.add(childPath);
            }
        }
        return childSchemaDirectories.build();
    } catch (IOException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

From source file:com.flipkart.fdp.migration.distcp.codec.GenericHadoopCodec.java

License:Apache License

public List<FileTuple> getFileStatusRecursive(Path path, Collection<String> excludeList)
        throws IOException, AuthenticationException {

    List<FileTuple> response = new ArrayList<FileTuple>();

    FileStatus file = fs.getFileStatus(path);
    //TODO excludeList to be checked if file (not folder) is mentioned in excludeList.
    if (file != null && file.isFile()) {
        response.add(new FileTuple(MirrorUtils.getSimplePath(file.getPath()), file.getLen(),
                file.getModificationTime()));
        return response;
    }/*from w ww . j  a  v  a  2s.  com*/

    FileStatus[] fstats = fs.listStatus(path);

    if (fstats != null && fstats.length > 0) {

        for (FileStatus fstat : fstats) {

            if (fstat.isDirectory() && !excludeList.contains(MirrorUtils.getSimplePath(fstat.getPath()))) {

                response.addAll(getFileStatusRecursive(fstat.getPath(), excludeList));
            } else {

                //TODO excludeList to be checked if file (not folder) is mentioned in excludeList.

                response.add(new FileTuple(MirrorUtils.getSimplePath(fstat.getPath()), fstat.getLen(),
                        fstat.getModificationTime()));
            }
        }
    }
    return response;
}