Example usage for org.apache.hadoop.fs FileStatus isDirectory

List of usage examples for org.apache.hadoop.fs FileStatus isDirectory

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory()

Source Link

Document

Is this a directory?

Usage

From source file:com.flipkart.fdp.migration.distcp.state.HDFSStateManager.java

License:Apache License

public Map<String, TransferStatus> getPreviousTransferStatus() throws IOException {

    Map<String, TransferStatus> status = new HashMap<String, TransferStatus>();
    FileStatus fstats[] = null;/*from  w  ww.  j  a v  a  2s  .  co m*/

    try {
        fstats = fs.listStatus(batchBasePath);
    } catch (Exception e) {
        System.out.println("No Previous states found: " + e.getMessage());
    }

    if (fstats == null || fstats.length <= 0)
        return status;

    List<FileStatus> fstatList = new ArrayList<FileStatus>();

    for (FileStatus fstat : fstats) {
        if (fstat.isDirectory())
            fstatList.add(fstat);
    }

    Collections.sort(fstatList, new Comparator<FileStatus>() {

        @Override
        public int compare(FileStatus o1, FileStatus o2) {
            return (int) (o2.getModificationTime() - o1.getModificationTime());
            // decending order sort by timestamp
        }
    });

    // ignore the current state folder as well.
    fstatList.remove(0);

    for (FileStatus fstat : fstatList) {
        System.out.println("Processing State History: " + fstat.getPath());

        Path spath = new Path(fstat.getPath(), PREVIOUS_STATE_FILE_NAME);
        List<TransferStatus> stats = getAllStats(new Path(fstat.getPath(), REPORT_PATH));
        mergeStates(status, stats);
        if (fs.exists(spath)) {
            stats = getAllStats(spath);
            mergeStates(status, stats);
            break;
        }
    }
    return status;
}

From source file:com.flipkart.fdp.migration.distcp.utils.FileCountDriver.java

License:Apache License

public List<String> getFileStatusRecursive(Path path, FileSystem fs, String destBasePath) throws IOException {
    List<String> response = new ArrayList<String>();
    FileStatus file = fs.getFileStatus(path);
    if (file != null && file.isFile()) {
        response.add(trimExtension(file.getPath().toUri().getPath(), destBasePath));
        return response;
    }/*from  w  ww  .  j a v  a2 s  .  c  om*/

    FileStatus[] fstats = fs.listStatus(path);

    if (fstats != null && fstats.length > 0) {

        for (FileStatus fstat : fstats) {

            if (fstat.isDirectory()) {
                response.addAll(getFileStatusRecursive(fstat.getPath(), fs, destBasePath));
            } else {
                response.add(trimExtension(fstat.getPath().toUri().getPath(), destBasePath));
            }
        }
    }
    return response;
}

From source file:com.flipkart.fdp.migration.distcp.utils.FileStatsDriver.java

License:Apache License

public List<String> getAllFilePath(Path filePath) throws IOException {
    List<String> fileList = new ArrayList<String>();
    FileStatus[] fileStatus = fs.listStatus(filePath);
    for (FileStatus fileStat : fileStatus) {
        if (fileStat.isDirectory()) {
            fileList.addAll(getAllFilePath(fileStat.getPath()));
        } else {/*from   w ww . j  ava  2s.  c  o m*/
            long ts = fileStat.getModificationTime();
            if (ts >= startTS && ts <= endTS)
                fileList.add(fileStat.getPath().toUri().getPath() + "," + fileStat.getLen());
        }
    }
    return fileList;
}

From source file:com.flipkart.fdp.migration.distcp.utils.HistoricFileCleanUpDriver.java

License:Apache License

public List<String> getAllFilePath(Path filePath) throws IOException {
    List<String> fileList = new ArrayList<String>();
    FileStatus[] fileStatus = fs.listStatus(filePath);
    for (FileStatus fileStat : fileStatus) {
        if (fileStat.isDirectory()) {
            if (fileStat.getModificationTime() >= startTS && fileStat.getModificationTime() <= endTS
                    && filePath.toUri().getPath().toString() != rootpath)
                fileList.add(fileStat.getPath().toUri().getPath());
            else//www  .java2s.  com
                fileList.addAll(getAllFilePath(fileStat.getPath()));
        }
        // } else {
        // if (fileStat.getModificationTime() >= startTS
        // && fileStat.getModificationTime() <= endTS)
        // fileList.add(fileStat.getPath().toUri().getPath());
        // }
    }
    return fileList;
}

From source file:com.flipkart.fdp.migration.distcp.utils.ZeroFileCleanUpDriver.java

License:Apache License

public List<String> getAllFilePath(Path filePath) throws IOException {
    List<String> fileList = new ArrayList<String>();
    FileStatus[] fileStatus = fs.listStatus(filePath);
    for (FileStatus fileStat : fileStatus) {
        if (fileStat.isDirectory()) {
            fileList.addAll(getAllFilePath(fileStat.getPath()));
        } else {//ww  w  .j  a va  2s.c  o  m
            if (fileStat.getLen() == 0)
                fileList.add(fileStat.getPath().toUri().getPath());
        }
    }
    return fileList;
}

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.HoplogUtil.java

License:Apache License

public static Collection<Collection<FileStatus>> getBucketHoplogs(Path regionPath, FileSystem fs, String type,
        long start, long end) throws IOException {
    Collection<Collection<FileStatus>> allBuckets = new ArrayList<Collection<FileStatus>>();

    // hoplog files names follow this pattern
    String HOPLOG_NAME_REGEX = AbstractHoplogOrganizer.HOPLOG_NAME_REGEX + type;
    String EXPIRED_HOPLOG_NAME_REGEX = HOPLOG_NAME_REGEX + AbstractHoplogOrganizer.EXPIRED_HOPLOG_EXTENSION;
    final Pattern pattern = Pattern.compile(HOPLOG_NAME_REGEX);
    final Pattern expiredPattern = Pattern.compile(EXPIRED_HOPLOG_NAME_REGEX);

    Path cleanUpIntervalPath = new Path(regionPath.getParent(), HoplogConfig.CLEAN_UP_INTERVAL_FILE_NAME);
    long intervalDurationMillis = readCleanUpIntervalMillis(fs, cleanUpIntervalPath);

    // a region directory contains directories for individual buckets. A bucket
    // has a integer name.
    FileStatus[] bucketDirs = fs.listStatus(regionPath);

    for (FileStatus bucket : bucketDirs) {
        if (!bucket.isDirectory()) {
            continue;
        }/*from  w  w w . j a  v a  2  s .  co m*/
        try {
            Integer.valueOf(bucket.getPath().getName());
        } catch (NumberFormatException e) {
            continue;
        }

        ArrayList<FileStatus> bucketHoplogs = new ArrayList<FileStatus>();

        // identify all the flush hoplogs and seq hoplogs by visiting all the
        // bucket directories
        FileStatus[] bucketFiles = fs.listStatus(bucket.getPath());

        Map<String, Long> expiredHoplogs = getExpiredHoplogs(fs, bucketFiles, expiredPattern);

        FileStatus oldestHopAfterEndTS = null;
        long oldestHopTS = Long.MAX_VALUE;
        long currentTimeStamp = System.currentTimeMillis();
        for (FileStatus file : bucketFiles) {
            if (!file.isFile()) {
                continue;
            }

            Matcher match = pattern.matcher(file.getPath().getName());
            if (!match.matches()) {
                continue;
            }

            long timeStamp = AbstractHoplogOrganizer.getHoplogTimestamp(match);
            if (start > 0 && timeStamp < start) {
                // this hoplog contains records less than the start time stamp
                continue;
            }

            if (end > 0 && timeStamp > end) {
                // this hoplog contains records mutated after end time stamp. Ignore
                // this hoplog if it is not the oldest.
                if (oldestHopTS > timeStamp) {
                    oldestHopTS = timeStamp;
                    oldestHopAfterEndTS = file;
                }
                continue;
            }
            long expiredTimeStamp = expiredTime(file, expiredHoplogs);
            if (expiredTimeStamp > 0 && intervalDurationMillis > 0) {
                if ((currentTimeStamp - expiredTimeStamp) > 0.8 * intervalDurationMillis) {
                    continue;
                }
            }
            bucketHoplogs.add(file);
        }

        if (oldestHopAfterEndTS != null) {
            long expiredTimeStamp = expiredTime(oldestHopAfterEndTS, expiredHoplogs);
            if (expiredTimeStamp <= 0 || intervalDurationMillis <= 0
                    || (currentTimeStamp - expiredTimeStamp) <= 0.8 * intervalDurationMillis) {
                bucketHoplogs.add(oldestHopAfterEndTS);
            }
        }

        if (bucketHoplogs.size() > 0) {
            allBuckets.add(bucketHoplogs);
        }
    }

    return allBuckets;
}

From source file:com.github.joshelser.accumulo.DelimitedIngest.java

License:Apache License

private List<Path> convertInputToPaths() throws IOException {
    List<String> inputs = args.getInput();
    List<Path> paths = new ArrayList<>(inputs.size());
    for (String input : inputs) {
        Path p = new Path(input);
        FileSystem fs = p.getFileSystem(conf);
        FileStatus fstat = fs.getFileStatus(p);
        if (fstat.isFile()) {
            paths.add(p);/*from  w ww . j  a  va2 s.c o m*/
        } else if (fstat.isDirectory()) {
            for (FileStatus child : fs.listStatus(p)) {
                if (child.isFile()) {
                    paths.add(child.getPath());
                }
            }
        } else {
            throw new IllegalStateException("Unable to handle that which is not file nor directory: " + p);
        }
    }
    return paths;
}

From source file:com.github.sadikovi.hadoop.riff.RiffOutputCommitter.java

License:Open Source License

private static void recurListFiles(FileSystem fs, FileStatus fileStatus, List<FileStatus> foundFiles,
        boolean fetchOneFile) throws IOException {
    if (fetchOneFile && !foundFiles.isEmpty())
        return;/* ww w.j ava  2 s .com*/
    if (fileStatus.isDirectory()) {
        FileStatus[] list = fs.listStatus(fileStatus.getPath(), PartFileFilter.instance);
        for (int i = 0; i < list.length; i++) {
            recurListFiles(fs, list[i], foundFiles, fetchOneFile);
        }
    } else {
        // file status is a file, add to the list
        foundFiles.add(fileStatus);
    }
}

From source file:com.google.mr4c.sources.HadoopFileSource.java

License:Open Source License

private List<String> getAllFileNames(Path dir) throws IOException {
    List<String> names = new ArrayList<String>();
    FileStatus[] files = m_fs.listStatus(dir);
    if (files == null) {
        throw new FileNotFoundException(String.format("[%s] is not an existing directory", dir));
    }//  w w w  .  j  a v  a  2 s.c  om
    for (FileStatus status : files) {
        if (status.isDirectory() && !m_flat) {
            names.addAll(getAllFileNames(status.getPath()));
        } else {
            String name = m_dir.toUri().relativize(status.getPath().toUri()).getPath();
            names.add(name);
        }
    }
    return names;
}

From source file:com.hdfstoftp.main.HdfsToFtp.java

/**
 * ?/*from  w  ww. j  a v a  2  s  .c  om*/
 * 
 * @param srcFS
 *            
 * @param src
 *            ?
 * @param dst
 *            
 * @param queryStr
 *            
 * @param deleteSource
 *            ??
 * @param overwrite
 *            ????
 * @return boolean
 * @throws Exception
 */
private static boolean copyFromHDFSToFTP(Config config) throws Exception {
    // ?hdfs
    Configuration conf = new Configuration();
    FileSystem srcFS = FileSystem.get(conf);
    long start = System.currentTimeMillis();
    boolean isRename = config.isRenameUploaded();
    int retryTimes = config.getRetryTimes();
    // ?
    String dstPath = config.getDestDir();
    Path src = new Path(config.getSouceDir());
    FileStatus fileStatus = srcFS.getFileStatus(src);
    String subDir = null;
    if (fileStatus.isDirectory()) {// 
        if (isRename) {// ??rename
            subDir = Config.RENAME_DIR;
            srcFS.mkdirs(new Path(fileStatus.getPath(), subDir));
        }
        int threadNum = config.getThreadNum();
        // 
        ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
        // ?ftp
        FTPClientPool ftpPool = new FTPClientPool(threadNum, new FtpClientFactory(config.getFTPClientConfig()));
        FTPClient ftpClient = ftpPool.borrowObject();
        // ?
        ftpClient.makeDirectory(dstPath);
        ftpPool.returnObject(ftpClient);
        // ??
        FileStatus contents[] = srcFS.listStatus(src);
        long beginFilter = 0;
        long endFileter = 0;

        if (config.getCommandLine().hasOption("d") || config.getCommandLine().hasOption("h")
                || config.getCommandLine().hasOption("t")) {// ?"["
            beginFilter = System.currentTimeMillis();
            Long[] timeRange = parseTimeRange(config.getCommandLine());
            contents = getNewContents(timeRange, contents);
            endFileter = System.currentTimeMillis();
        }
        // ?
        if (config.getCommandLine().hasOption("r")) {// "["??
            beginFilter = System.currentTimeMillis();
            contents = getFilterContents(config.getCommandLine().getOptionValue("r").trim(), contents);
            endFileter = System.currentTimeMillis();
        }
        logger.info("total file count:" + contents.length);
        Map<String, String> fileNameMap = null;
        long beginSkip = 0;
        long endSkip = 0;
        boolean overwrite = true;
        if (config.getCommandLine().hasOption("o")) {
            overwrite = "true".equals(config.getCommandLine().getOptionValue("o").trim());
        }
        if (!overwrite) {// ?????
            beginSkip = System.currentTimeMillis();
            fileNameMap = getFileNameMap(dstPath, ftpPool);
            endSkip = System.currentTimeMillis();
        }
        int skiped = 0;

        List<Future<?>> futureList = new ArrayList<Future<?>>();
        for (int i = 0; i < contents.length; i++) {
            if (!overwrite && fileNameMap.containsKey(contents[i].getPath().getName())) {
                // 
                skiped++;
                Log.info("skiped filename:" + contents[i].getPath().getName());
                continue;
            }
            if (contents[i].isDirectory()) {
                continue;
            }
            // ???
            Future<?> future = threadPool.submit(new UploadFileTask(srcFS, contents[i].getPath(),
                    new Path(dstPath, contents[i].getPath().getName()), ftpPool, false, isRename, subDir,
                    retryTimes));
            futureList.add(future);
        }
        int transfered = 0;
        int failed = 0;
        for (Future<?> future : futureList) {
            Boolean computeResult = (Boolean) future.get();
            if (computeResult) {
                transfered++;
                if (transfered % 50 == 0 || transfered == contents.length) {
                    logger.info("have transfered:" + transfered + " files");
                }
            } else {
                failed++;
                logger.error("failed transter:" + failed + " files");
            }
        }
        // 
        threadPool.shutdown();
        // FTPCient
        ftpPool.close();
        // ****************
        logger.info("filter time:" + (endFileter - beginFilter) + " ms");
        if (!overwrite) {
            logger.info("skip time:" + (endSkip - beginSkip) + " ms");
        }
        logger.info("total file count:" + contents.length);
        logger.info("total transtered: " + transfered + ",total failed:" + failed + ",total skiped:" + skiped);

    } else {// 

        BufferedReader reader = null;
        FtpClientFactory facotry = new FtpClientFactory(config.getFTPClientConfig());
        FTPClient ftpClient = null;
        InputStream in = null;
        try {
            Path path = fileStatus.getPath();
            if (!path.getName().contains("log")) {

            }
            reader = new BufferedReader(new FileReader(new File(path.toUri().getPath())));
            String str = null;

            ftpClient = facotry.makeObject();

            while ((str = reader.readLine()) != null) {
                String[] feilds = str.split("&");
                Path filePath = null;
                if (feilds.length == 2 && feilds[1] != "") {
                    filePath = new Path(feilds[1]);
                    in = srcFS.open(filePath);
                    boolean result = ftpClient.storeFile(dstPath, in);
                    System.out.println(ftpClient.getReplyCode());
                    if (result) {
                        logger.info(filePath.toString());
                    } else {
                        logger_failed.info(filePath.toString());
                    }
                } else {
                    continue;
                }

            }
        } catch (Exception e) {
            e.printStackTrace();

        } finally {
            in.close();
            reader.close();
            facotry.destroyObject(ftpClient);
        }

    }
    long end = System.currentTimeMillis();
    logger.info("finished transfer,total time:" + (end - start) / 1000 + "s");
    return true;
}