Example usage for org.apache.hadoop.fs FileStatus isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory() 

Source Link

Document

Is this a directory?

Usage

From source file:hdfs.jsr203.HadoopFileSystem.java

License:Apache License

void copyFile(boolean deletesrc, byte[] src, byte[] dst, CopyOption... options) throws IOException {
    checkWritable();/*from   ww  w  .j  a v  a 2s  . com*/
    if (Arrays.equals(src, dst))
        return; // do nothing, src and dst are the same

    beginWrite();
    try {
        ensureOpen();
        org.apache.hadoop.fs.Path eSrc_path = new HadoopPath(this, src).getRawResolvedPath();
        FileStatus eSrc = this.fs.getFileStatus(eSrc_path);
        if (!this.fs.exists(eSrc_path))
            throw new NoSuchFileException(getString(src));
        if (eSrc.isDirectory()) { // specification says to create dst directory
            createDirectory(dst);
            return;
        }
        boolean hasReplace = false;
        boolean hasCopyAttrs = false;
        for (CopyOption opt : options) {
            if (opt == REPLACE_EXISTING)
                hasReplace = true;
            else if (opt == COPY_ATTRIBUTES)
                hasCopyAttrs = true;
        }
        org.apache.hadoop.fs.Path eDst_path = new HadoopPath(this, dst).getRawResolvedPath();
        //            FileStatus eDst = this.fs.getFileStatus(eDst_path); //if eDst_path not exist, it will throw an error

        if (fs.exists(eDst_path)) {
            if (!hasReplace)
                throw new FileAlreadyExistsException(getString(dst));

            if (!fs.delete(eDst_path, false)) {
                throw new AccessDeniedException("cannot delete hdfs file " + getString(dst));
            }
        } else {
            //checkParents(dst);
        }

        //Simply use FileUtil.copy here. Can we use DistCp for very big files here? zongjie@novelbio.com
        boolean isCanDeleteSourceFile = FileUtil.copy(fs, eSrc_path, fs, eDst_path, deletesrc, fs.getConf());
        if (!isCanDeleteSourceFile) {
            throw new AccessDeniedException("cannot delete source file " + eSrc_path.toString());
        }

        //            org.apache.hadoop.fs.Path[] srcs = new org.apache.hadoop.fs.Path[] {eSrc_path};
        //         this.fs.concat(eDst_path, srcs);

        /*
        Entry u = new Entry(eSrc, Entry.COPY);    // copy eSrc entry
        u.name(dst);                              // change name
        if (eSrc.type == Entry.NEW || eSrc.type == Entry.FILECH)
        {
        u.type = eSrc.type;    // make it the same type
        if (!deletesrc) {      // if it's not "rename", just take the data
            if (eSrc.bytes != null)
                u.bytes = Arrays.copyOf(eSrc.bytes, eSrc.bytes.length);
            else if (eSrc.file != null) {
                u.file = getTempPathForEntry(null);
                Files.copy(eSrc.file, u.file, REPLACE_EXISTING);
            }
        }
        }
        if (!hasCopyAttrs)
        u.mtime = u.atime= u.ctime = System.currentTimeMillis();
        update(u);
        if (deletesrc)
        updateDelete(eSrc);*/
    } finally {
        endWrite();
    }
}

---

From source file:hdfs.jsr203.HadoopFileSystem.java

License:Apache License

void moveFile(byte[] src, byte[] dst, CopyOption... options) throws IOException {
    checkWritable();/*from ww w .j a  va2  s  .c o m*/
    if (Arrays.equals(src, dst))
        return; // do nothing, src and dst are the same

    beginWrite();
    try {
        ensureOpen();
        org.apache.hadoop.fs.Path eSrc_path = new HadoopPath(this, src).getRawResolvedPath();
        FileStatus eSrc = this.fs.getFileStatus(eSrc_path);
        if (!this.fs.exists(eSrc_path))
            throw new NoSuchFileException(getString(src));
        if (eSrc.isDirectory()) { // specification says to create dst directory
            createDirectory(dst);
            return;
        }
        boolean hasReplace = false;
        boolean hasCopyAttrs = false;
        for (CopyOption opt : options) {
            if (opt == REPLACE_EXISTING)
                hasReplace = true;
            else if (opt == COPY_ATTRIBUTES)
                hasCopyAttrs = true;
        }
        org.apache.hadoop.fs.Path eDst_path = new HadoopPath(this, dst).getRawResolvedPath();

        if (fs.exists(eDst_path)) {
            if (!hasReplace)
                throw new FileAlreadyExistsException(getString(dst));

            if (!fs.delete(eDst_path, false)) {
                throw new AccessDeniedException("cannot delete hdfs file " + getString(dst));
            }
        }
        //Simply rename the path
        if (!fs.rename(eSrc_path, eDst_path)) {
            throw new AccessDeniedException("cannot move source file " + eSrc_path.toString());
        }

    } finally {
        endWrite();
    }
}

---

From source file:info.halo9pan.word2vec.hadoop.mr.SortOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext job) throws InvalidJobConfException, IOException {
    // Ensure that the output directory is set
    Path outDir = getOutputPath(job);
    if (outDir == null) {
        throw new InvalidJobConfException("Output directory not set in JobConf.");
    }/* w ww .j av  a 2s .c o  m*/

    final Configuration jobConf = job.getConfiguration();

    // get delegation token for outDir's file system
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, jobConf);

    final FileSystem fs = outDir.getFileSystem(jobConf);

    if (fs.exists(outDir)) {
        // existing output dir is considered empty iff its only content is the
        // partition file.
        //
        final FileStatus[] outDirKids = fs.listStatus(outDir);
        boolean empty = false;
        if (outDirKids != null && outDirKids.length == 1) {
            final FileStatus st = outDirKids[0];
            final String fname = st.getPath().getName();
            empty = !st.isDirectory() && SortInputFormat.PARTITION_FILENAME.equals(fname);
        }
        if (WordSort.getUseSimplePartitioner(job) || !empty) {
            throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
        }
    }
}

---

From source file:info.halo9pan.word2vec.hadoop.terasort.TeraOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext job) throws InvalidJobConfException, IOException {
    // Ensure that the output directory is set
    Path outDir = getOutputPath(job);
    if (outDir == null) {
        throw new InvalidJobConfException("Output directory not set in JobConf.");
    }//from  ww  w.  ja  v  a2s  .co  m

    final Configuration jobConf = job.getConfiguration();

    // get delegation token for outDir's file system
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, jobConf);

    final FileSystem fs = outDir.getFileSystem(jobConf);

    if (fs.exists(outDir)) {
        // existing output dir is considered empty iff its only content is the
        // partition file.
        //
        final FileStatus[] outDirKids = fs.listStatus(outDir);
        boolean empty = false;
        if (outDirKids != null && outDirKids.length == 1) {
            final FileStatus st = outDirKids[0];
            final String fname = st.getPath().getName();
            empty = !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
        }
        if (TeraSort.getUseSimplePartitioner(job) || !empty) {
            throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
        }
    }
}

---

From source file:io.confluent.connect.hdfs.FileUtils.java

License:Apache License

private static ArrayList<FileStatus> traverseImpl(Storage storage, Path path, PathFilter filter)
        throws IOException {
    if (!storage.exists(path.toString())) {
        return new ArrayList<>();
    }/*w  w w.ja  va 2  s . c o m*/
    ArrayList<FileStatus> result = new ArrayList<>();
    FileStatus[] statuses = storage.listStatus(path.toString());
    for (FileStatus status : statuses) {
        if (status.isDirectory()) {
            result.addAll(traverseImpl(storage, status.getPath(), filter));
        } else {
            if (filter.accept(status.getPath())) {
                result.add(status);
            }
        }
    }
    return result;
}

---

From source file:io.confluent.connect.hdfs.FileUtils.java

License:Apache License

public static FileStatus fileStatusWithMaxOffset(Storage storage, Path path, CommittedFileFilter filter)
        throws IOException {
    if (!storage.exists(path.toString())) {
        return null;
    }//ww  w.  j a  v a  2s  .  c  om
    long maxOffset = -1L;
    FileStatus fileStatusWithMaxOffset = null;
    FileStatus[] statuses = storage.listStatus(path.toString());
    for (FileStatus status : statuses) {
        if (status.isDirectory()) {
            FileStatus fileStatus = fileStatusWithMaxOffset(storage, status.getPath(), filter);
            if (fileStatus != null) {
                long offset = extractOffset(fileStatus.getPath().getName());
                if (offset > maxOffset) {
                    maxOffset = offset;
                    fileStatusWithMaxOffset = fileStatus;
                }
            }
        } else {
            String filename = status.getPath().getName();
            log.trace("Checked for max offset: {}", status.getPath());
            if (filter.accept(status.getPath())) {
                long offset = extractOffset(filename);
                if (offset > maxOffset) {
                    maxOffset = offset;
                    fileStatusWithMaxOffset = status;
                }
            }
        }
    }
    return fileStatusWithMaxOffset;
}

---

From source file:io.confluent.connect.hdfs.FileUtils.java

License:Apache License

private static ArrayList<FileStatus> getDirectoriesImpl(Storage storage, Path path) throws IOException {
    FileStatus[] statuses = storage.listStatus(path.toString());
    ArrayList<FileStatus> result = new ArrayList<>();
    for (FileStatus status : statuses) {
        if (status.isDirectory()) {
            int count = 0;
            FileStatus[] fileStatuses = storage.listStatus(status.getPath().toString());
            for (FileStatus fileStatus : fileStatuses) {
                if (fileStatus.isDirectory()) {
                    result.addAll(getDirectoriesImpl(storage, fileStatus.getPath()));
                } else {
                    count++;/*from  ww w  .j av a  2 s  . co m*/
                }
            }
            if (count == fileStatuses.length) {
                result.add(status);
            }
        }
    }
    return result;
}

---

From source file:io.confluent.connect.hdfs.FileUtils.java

License:Apache License

private static ArrayList<FileStatus> traverseImpl(FileSystem fs, Path path) throws IOException {
    if (!fs.exists(path)) {
        return new ArrayList<>();
    }/*  www. j  a  v  a2 s.c  o m*/
    ArrayList<FileStatus> result = new ArrayList<>();
    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
        if (status.isDirectory()) {
            result.addAll(traverseImpl(fs, status.getPath()));
        } else {
            result.add(status);
        }
    }
    return result;
}

---

From source file:io.prestosql.plugin.hive.AbstractTestHiveClient.java

License:Apache License

protected Set<String> listAllDataFiles(HdfsContext context, Path path) throws IOException {
    Set<String> result = new HashSet<>();
    FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, path);
    if (fileSystem.exists(path)) {
        for (FileStatus fileStatus : fileSystem.listStatus(path)) {
            if (fileStatus.getPath().getName().startsWith(".presto")) {
                // skip hidden files
            } else if (fileStatus.isFile()) {
                result.add(fileStatus.getPath().toString());
            } else if (fileStatus.isDirectory()) {
                result.addAll(listAllDataFiles(context, fileStatus.getPath()));
            }//w w  w.j  a v a2  s.  c  om
        }
    }
    return result;
}

---

From source file:io.prestosql.plugin.hive.metastore.file.FileHiveMetastore.java

License:Apache License

private List<ArrayDeque<String>> listPartitions(Path director, List<Column> partitionColumns) {
    if (partitionColumns.isEmpty()) {
        return ImmutableList.of();
    }/*ww  w .j  a v  a  2s  .  c  om*/

    try {
        String directoryPrefix = partitionColumns.get(0).getName() + '=';

        List<ArrayDeque<String>> partitionValues = new ArrayList<>();
        for (FileStatus fileStatus : metadataFileSystem.listStatus(director)) {
            if (!fileStatus.isDirectory()) {
                continue;
            }
            if (!fileStatus.getPath().getName().startsWith(directoryPrefix)) {
                continue;
            }

            List<ArrayDeque<String>> childPartitionValues;
            if (partitionColumns.size() == 1) {
                childPartitionValues = ImmutableList.of(new ArrayDeque<>());
            } else {
                childPartitionValues = listPartitions(fileStatus.getPath(),
                        partitionColumns.subList(1, partitionColumns.size()));
            }

            String value = unescapePathName(fileStatus.getPath().getName().substring(directoryPrefix.length()));
            for (ArrayDeque<String> childPartition : childPartitionValues) {
                childPartition.addFirst(value);
                partitionValues.add(childPartition);
            }
        }
        return partitionValues;
    } catch (IOException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, "Error listing partition directories", e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »