Example usage for org.apache.hadoop.fs FileStatus isDirectory

List of usage examples for org.apache.hadoop.fs FileStatus isDirectory

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory()

Source Link

Document

Is this a directory?

Usage

From source file:org.apache.falcon.extensions.store.ExtensionStore.java

License:Apache License

private List<String> getTrustedExtensions() throws StoreAccessException {
    List<String> extensionList = new ArrayList<>();
    try {//from   w w  w.jav  a2s.c o  m
        FileStatus[] fileStatuses = fs.listStatus(storePath);

        for (FileStatus fileStatus : fileStatuses) {
            if (fileStatus.isDirectory()) {
                Path filePath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
                extensionList.add(filePath.getName());
            }
        }
    } catch (IOException e) {
        throw new StoreAccessException(e);
    }
    return extensionList;
}

From source file:org.apache.falcon.hadoop.JailedFileSystem.java

License:Apache License

@Override
public FileStatus[] listStatus(Path f) throws IOException {
    FileStatus[] fileStatuses = localFS.listStatus(toLocalPath(f));
    if (fileStatuses == null || fileStatuses.length == 0) {
        return fileStatuses;
    } else {// ww  w  . j a v  a 2 s  .c  o m
        FileStatus[] jailFileStatuses = new FileStatus[fileStatuses.length];
        for (int index = 0; index < fileStatuses.length; index++) {
            FileStatus status = fileStatuses[index];
            jailFileStatuses[index] = new FileStatus(status.getLen(), status.isDirectory(),
                    status.getReplication(), status.getBlockSize(), status.getModificationTime(),
                    status.getAccessTime(), status.getPermission(), status.getOwner(), status.getGroup(),
                    fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory()));
        }
        return jailFileStatuses;
    }
}

From source file:org.apache.falcon.hadoop.JailedFileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path f) throws IOException {
    FileStatus status = localFS.getFileStatus(toLocalPath(f));
    if (status == null) {
        return null;
    }/*from   ww w .j  av  a2 s  .c  om*/
    return new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(), status.getBlockSize(),
            status.getModificationTime(), status.getAccessTime(), status.getPermission(), status.getOwner(),
            status.getGroup(),
            fromLocalPath(status.getPath()).makeQualified(this.getUri(), this.getWorkingDirectory()));
}

From source file:org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils.java

License:Apache License

/**
 * Adds path(will be the list of directories containing jars to be added as external jars to workflow e.g.
 * for feeds libext, libext/FEED/, libext/FEED/RETENTION, libext/FEED/REPLICATION as an extension jar to the
 * workflow. e.g./*from w  w  w . ja  v a2  s. c o m*/
 *
 * @param fs
 * @param path
 * @param wf
 * @throws IOException
 */
public static void addExtensionJars(FileSystem fs, Path path, WORKFLOWAPP wf) throws IOException {
    FileStatus[] libs;
    try {
        libs = fs.listStatus(path);
    } catch (FileNotFoundException ignore) {
        //Ok if the libext is not configured
        return;
    }

    for (FileStatus lib : libs) {
        if (lib.isDirectory()) {
            continue;
        }

        for (Object obj : wf.getDecisionOrForkOrJoin()) {
            if (!(obj instanceof ACTION)) {
                continue;
            }
            ACTION action = (ACTION) obj;
            List<String> files = null;
            if (action.getJava() != null) {
                files = action.getJava().getFile();
            } else if (action.getPig() != null) {
                files = action.getPig().getFile();
            } else if (action.getMapReduce() != null) {
                files = action.getMapReduce().getFile();
            }
            if (files != null) {
                files.add(lib.getPath().toString());
            }
        }
    }
}

From source file:org.apache.falcon.oozie.feed.OozieFeedWorkflowBuilderTest.java

License:Apache License

private void verifyWorkflowUMask(FileSystem fs, COORDINATORAPP coord, String defaultUMask) throws IOException {
    Assert.assertEquals(fs.getConf().get("fs.permissions.umask-mode"), defaultUMask);

    String appPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
    Path wfPath = new Path(appPath);
    FileStatus[] fileStatuses = fs.listStatus(wfPath);
    for (FileStatus fileStatus : fileStatuses) {
        Assert.assertEquals(fileStatus.getOwner(), CurrentUser.getProxyUGI().getShortUserName());

        final FsPermission permission = fileStatus.getPermission();
        if (!fileStatus.isDirectory()) {
            Assert.assertEquals(permission.toString(),
                    HadoopClientFactory.getFileDefaultPermission(fs.getConf()).toString());
        }/*ww  w  .j  ava 2 s.  c o m*/
    }
}

From source file:org.apache.falcon.oozie.OozieOrchestrationWorkflowBuilder.java

License:Apache License

private void addExtensionJars(FileSystem fs, Path path, WORKFLOWAPP wf) throws IOException {
    FileStatus[] libs = null;/*w  w w  . j  a  va2  s  .c  o  m*/
    try {
        libs = fs.listStatus(path);
    } catch (FileNotFoundException ignore) {
        //Ok if the libext is not configured
    }

    if (libs == null) {
        return;
    }

    for (FileStatus lib : libs) {
        if (lib.isDirectory()) {
            continue;
        }

        for (Object obj : wf.getDecisionOrForkOrJoin()) {
            if (!(obj instanceof ACTION)) {
                continue;
            }
            ACTION action = (ACTION) obj;
            List<String> files = null;
            if (action.getJava() != null) {
                files = action.getJava().getFile();
            } else if (action.getPig() != null) {
                files = action.getPig().getFile();
            } else if (action.getMapReduce() != null) {
                files = action.getMapReduce().getFile();
            }
            if (files != null) {
                files.add(lib.getPath().toString());
            }
        }
    }
}

From source file:org.apache.giraph.yarn.GiraphYarnClient.java

License:Apache License

/**
 * Without Hadoop MR to check for us, make sure the output dir doesn't exist!
 *///from   w  ww  .j  a  v a 2s.  c  o  m
private void verifyOutputDirDoesNotExist() {
    Path outDir = null;
    try {
        FileSystem fs = FileSystem.get(giraphConf);
        String errorMsg = "__ERROR_NO_OUTPUT_DIR_SET__";
        outDir = new Path(fs.getHomeDirectory(), giraphConf.get(OUTDIR, errorMsg));
        FileStatus outStatus = fs.getFileStatus(outDir);
        if (outStatus.isDirectory() || outStatus.isFile() || outStatus.isSymlink()) {
            throw new IllegalStateException("Path " + outDir + " already exists.");
        }
    } catch (IOException ioe) {
        LOG.info("Final output path is: " + outDir);
    }
}

From source file:org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore.java

License:Apache License

/**
 * Retrieves all the {@link ConfigKeyPath}s that are imported by the given {@link ConfigKeyPath}. This method does this
 * by reading the {@link #INCLUDES_CONF_FILE_NAME} file associated with the dataset specified by the given
 * {@link ConfigKeyPath}. If the {@link Path} described by the {@link ConfigKeyPath} does not exist, then an empty
 * {@link List} is returned./* w w  w . ja v a2 s.c o  m*/
 *
 * @param  configKey      the config key path whose tags are needed
 * @param  version        the configuration version in the configuration store.
 *
 * @return a {@link List} of {@link ConfigKeyPath}s where each entry is a {@link ConfigKeyPath} imported by the dataset
 * specified by the configKey.
 *
 * @throws VersionDoesNotExistException if the version specified cannot be found in the {@link ConfigStore}.
 */
public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version,
        Optional<Config> runtimeConfig) throws VersionDoesNotExistException {
    Preconditions.checkNotNull(configKey, "configKey cannot be null!");
    Preconditions.checkArgument(!Strings.isNullOrEmpty(version), "version cannot be null or empty!");

    List<ConfigKeyPath> configKeyPaths = new ArrayList<>();
    Path datasetDir = getDatasetDirForKey(configKey, version);
    Path includesFile = new Path(datasetDir, INCLUDES_CONF_FILE_NAME);

    try {
        if (!this.fs.exists(includesFile)) {
            return configKeyPaths;
        }

        FileStatus includesFileStatus = this.fs.getFileStatus(includesFile);
        if (!includesFileStatus.isDirectory()) {
            try (InputStream includesConfInStream = this.fs.open(includesFileStatus.getPath())) {
                configKeyPaths.addAll(getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig));
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey),
                e);
    }

    return configKeyPaths;
}

From source file:org.apache.gobblin.runtime.spec_store.FSSpecStore.java

License:Apache License

/**
 * For multiple {@link FlowSpec}s to be loaded, catch Exceptions when one of them failed to be loaded and
 * continue with the rest./*from w  ww  .  j a  va2 s.  c  om*/
 *
 * The {@link IOException} thrown from standard FileSystem call will be propagated, while the file-specific
 * exception will be caught to ensure other files being able to deserialized.
 *
 * @param directory The directory that contains specs to be deserialized
 * @param specs Container of specs.
 */
private void getSpecs(Path directory, Collection<Spec> specs) throws Exception {
    FileStatus[] fileStatuses = fs.listStatus(directory);
    for (FileStatus fileStatus : fileStatuses) {
        if (fileStatus.isDirectory()) {
            getSpecs(fileStatus.getPath(), specs);
        } else {
            try {
                specs.add(readSpecFromFile(fileStatus.getPath()));
            } catch (Exception e) {
                log.warn(String.format("Path[%s] cannot be correctly deserialized as Spec",
                        fileStatus.getPath()), e);
            }
        }
    }
}

From source file:org.apache.gobblin.source.RegexBasedPartitionedRetriever.java

License:Apache License

private List<FileInfo> getOuterDirectories(FileSystem fs, long minWatermark, long maxAllowedWatermark)
        throws IOException {
    LOGGER.debug("Listing contents of {}", sourceDir);

    FileStatus[] fileStatus = fs.listStatus(sourceDir);
    List<FileInfo> outerDirectories = new ArrayList<>();

    for (FileStatus file : fileStatus) {
        if (!file.isDirectory()) {
            LOGGER.debug("Skipping non-directory {}", file.getPath().toUri());
            continue;
        }/*from  www.j  a v a  2s. c  o m*/

        try {
            long watermark = getWatermarkFromString(extractWatermarkFromDirectory(file.getPath().getName()));
            if (watermark > minWatermark && watermark < maxAllowedWatermark) {
                LOGGER.info("Processing directory {} with watermark {}", file.getPath(), watermark);
                outerDirectories.add(new FileInfo(file.getPath().toString(), 0, watermark,
                        PathUtils.relativizePath(file.getPath(), sourceDir).toString()));
            } else {
                LOGGER.info(
                        "Ignoring directory {} - watermark {} is not between minWatermark {} and (now-leadTime) {}",
                        file.getPath(), watermark, minWatermark, maxAllowedWatermark);
            }
        } catch (IllegalArgumentException e) {
            LOGGER.info("Directory {} ({}) does not match pattern {}; skipping", file.getPath().getName(),
                    file.getPath(), this.pattern.toString());
        }
    }

    Collections.sort(outerDirectories);
    return outerDirectories;
}