Example usage for org.apache.hadoop.fs FileStatus isDirectory

List of usage examples for org.apache.hadoop.fs FileStatus isDirectory

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory()

Source Link

Document

Is this a directory?

Usage

From source file:AggregatedLogsPurger.java

License:Apache License

public boolean purge() throws IOException {
    LocalDateTime now = LocalDateTime.now();
    LocalDateTime deleteLogsOlderThanTime = now.minusDays(deleteOlderThanDays);

    //Identify which log dirs should be deleted
    FileSystem fs = rootLogDir.getFileSystem(conf);
    try {/*from   w  w  w . ja v  a  2  s. co  m*/

        long totalBytes = 0;
        for (FileStatus userDir : fs.listStatus(rootLogDir)) {
            if (userDir.isDirectory()) {
                Path userDirPath = new Path(userDir.getPath(), suffix);
                System.out.println("Checking for userDir : " + userDirPath);
                for (FileStatus appDir : fs.listStatus(userDirPath)) {
                    LocalDateTime appDirDate = getAppDirDateTime(appDir.getModificationTime());
                    if (appDirDate.isBefore(deleteLogsOlderThanTime)) {
                        long size = getLengthRecursively(fs, appDir.getPath());
                        System.out.println(appDir.getPath() + ", " + appDir.getOwner() + ", "
                                + appDirDate.toString() + ", size=" + size);
                        totalBytes += size;
                        if (shouldDelete) {
                            System.out.println("Deleting " + appDir.getPath());
                            fs.delete(appDir.getPath(), true);
                        }
                    }
                }
            }
        }
        System.out.println("Savings : " + totalBytes);
    } catch (IOException e) {
        e.printStackTrace();
        return false;
    } finally {
        fs.close();
    }
    return true;
}

From source file:AggregatedLogsPurger.java

License:Apache License

private long getLengthRecursively(FileSystem fs, Path path) throws IOException {
    long size = 0;
    for (FileStatus status : fs.listStatus(path)) {
        if (status.isDirectory()) {
            getLengthRecursively(fs, status.getPath());
        } else {/*from w w w  .j ava  2 s  .co  m*/
            size += status.getLen();
        }
    }
    return size;
}

From source file:be.ugent.intec.halvade.HalvadeOptions.java

License:Open Source License

protected double getInputSize(String input, Configuration conf) throws URISyntaxException, IOException {
    double size = 0;
    FileSystem fs = FileSystem.get(new URI(input), conf);
    if (fs.getFileStatus(new Path(input)).isDirectory()) {
        // add every file in directory
        FileStatus[] files = fs.listStatus(new Path(input));
        for (FileStatus file : files) {
            if (!file.isDirectory()) {
                size += file.getLen();/* www.  ja  v  a 2 s. com*/
            }
        }
    } else {
        size += fs.getFileStatus(new Path(input)).getLen();
    }
    return (size / (1024 * 1024 * 1024));
}

From source file:be.ugent.intec.halvade.MapReduceRunner.java

License:Open Source License

protected int runPass1RNAJob(Configuration pass1Conf, String tmpOutDir)
        throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException {
    HalvadeConf.setIsPass2(pass1Conf, false);
    HalvadeResourceManager.setJobResources(halvadeOpts, pass1Conf, HalvadeResourceManager.RNA_SHMEM_PASS1, true,
            halvadeOpts.useBamInput);/*from   www .  j  a  v  a  2  s. co  m*/
    Job pass1Job = Job.getInstance(pass1Conf, "Halvade pass 1 RNA pipeline");
    pass1Job.addCacheArchive(new URI(halvadeOpts.halvadeBinaries));
    pass1Job.setJarByClass(be.ugent.intec.halvade.hadoop.mapreduce.HalvadeMapper.class);
    FileSystem fs = FileSystem.get(new URI(halvadeOpts.in), pass1Conf);
    try {
        if (fs.getFileStatus(new Path(halvadeOpts.in)).isDirectory()) {
            // add every file in directory
            FileStatus[] files = fs.listStatus(new Path(halvadeOpts.in));
            for (FileStatus file : files) {
                if (!file.isDirectory()) {
                    FileInputFormat.addInputPath(pass1Job, file.getPath());
                }
            }
        } else {
            FileInputFormat.addInputPath(pass1Job, new Path(halvadeOpts.in));
        }
    } catch (IOException | IllegalArgumentException e) {
        Logger.EXCEPTION(e);
    }

    FileSystem outFs = FileSystem.get(new URI(tmpOutDir), pass1Conf);
    boolean skipPass1 = false;
    if (outFs.exists(new Path(tmpOutDir))) {
        // check if genome already exists
        skipPass1 = outFs.exists(new Path(tmpOutDir + "/_SUCCESS"));
        if (skipPass1)
            Logger.DEBUG("pass1 genome already created, skipping pass 1");
        else {
            Logger.INFO("The output directory \'" + tmpOutDir + "\' already exists.");
            Logger.INFO("ERROR: Please remove this directory before trying again.");
            System.exit(-2);
        }
    }
    if (!skipPass1) {
        FileOutputFormat.setOutputPath(pass1Job, new Path(tmpOutDir));
        pass1Job.setMapperClass(be.ugent.intec.halvade.hadoop.mapreduce.StarAlignPassXMapper.class);

        pass1Job.setInputFormatClass(HalvadeTextInputFormat.class);
        pass1Job.setMapOutputKeyClass(GenomeSJ.class);
        pass1Job.setMapOutputValueClass(Text.class);

        pass1Job.setSortComparatorClass(GenomeSJSortComparator.class);
        pass1Job.setGroupingComparatorClass(GenomeSJGroupingComparator.class);
        pass1Job.setNumReduceTasks(1);
        pass1Job.setReducerClass(be.ugent.intec.halvade.hadoop.mapreduce.RebuildStarGenomeReducer.class);
        pass1Job.setOutputKeyClass(LongWritable.class);
        pass1Job.setOutputValueClass(Text.class);

        return runTimedJob(pass1Job, "Halvade pass 1 Job");
    } else
        return 0;
}

From source file:be.ugent.intec.halvade.MapReduceRunner.java

License:Open Source License

protected void addInputFiles(String input, Configuration conf, Job job) throws URISyntaxException, IOException {
    FileSystem fs = FileSystem.get(new URI(input), conf);
    Logger.DEBUG("adding input files from " + input);
    if (fs.getFileStatus(new Path(input)).isDirectory()) {
        // add every file in directory
        FileStatus[] files = fs.listStatus(new Path(input));
        for (FileStatus file : files) {
            if (!file.isDirectory()) {
                FileInputFormat.addInputPath(job, file.getPath());
            }//w  w w.ja v  a  2 s  .c  o m
        }
    } else
        FileInputFormat.addInputPath(job, new Path(input));
}

From source file:be.ugent.intec.halvade.MapReduceRunner.java

License:Open Source License

protected void addInputFiles(String input, Configuration conf, Job job, String filter)
        throws URISyntaxException, IOException {
    FileSystem fs = FileSystem.get(new URI(input), conf);
    if (fs.getFileStatus(new Path(input)).isDirectory()) {
        // add every file in directory
        FileStatus[] files = fs.listStatus(new Path(input));
        for (FileStatus file : files) {
            if (!file.isDirectory() && file.getPath().getName().endsWith(filter)) {
                FileInputFormat.addInputPath(job, file.getPath());
            }/* ww  w  . j  av a2 s  .co m*/
        }
    } else {
        FileInputFormat.addInputPath(job, new Path(input));
    }
}

From source file:be.ugent.intec.halvade.utils.HalvadeConf.java

License:Open Source License

public static int getMapTasksLeft(Configuration conf) throws IOException, URISyntaxException {
    int containers = conf.getInt(totalContainers, 1);
    int tasks = 0;
    String filedir = conf.get(outdir) + tasksDone;
    FileSystem fs = FileSystem.get(new URI(filedir), conf);
    FileStatus[] files = fs.listStatus(new Path(filedir));
    for (FileStatus file : files) {
        if (!file.isDirectory()) {
            tasks++;/*from   w  w w.  j  a v  a2s .  com*/
        }
    }
    Logger.DEBUG("containers left: " + (Integer.parseInt(conf.get("mapred.map.tasks")) - tasks));
    return Integer.parseInt(conf.get("mapred.map.tasks")) - tasks;
}

From source file:be.ugent.intec.halvade.utils.HalvadeConf.java

License:Open Source License

public static boolean allTasksCompleted(Configuration conf) throws IOException, URISyntaxException {
    int tasks = 0;
    String filedir = conf.get(outdir) + tasksDone;
    FileSystem fs = FileSystem.get(new URI(filedir), conf);
    FileStatus[] files = fs.listStatus(new Path(filedir));
    for (FileStatus file : files) {
        if (!file.isDirectory()) {
            tasks++;/*from  w w  w  .ja va 2 s. com*/
        }
    }
    Logger.DEBUG("tasks started: " + tasks);
    return tasks >= Integer.parseInt(conf.get("mapred.map.tasks"));
}

From source file:ch.cern.db.hdfs.Main.java

License:GNU General Public License

private void printFileStatus(FileStatus status) {
    System.out.println();/*from  www  .  jav a2 s .  co  m*/
    System.out.println("Showing metadata for: " + status.getPath());
    System.out.println("   isDirectory: " + status.isDirectory());
    System.out.println("   isFile: " + status.isFile());
    System.out.println("   isSymlink: " + status.isSymlink());
    System.out.println("   encrypted: " + status.isEncrypted());
    System.out.println("   length: " + status.getLen());
    System.out.println("   replication: " + status.getReplication());
    System.out.println("   blocksize: " + status.getBlockSize());
    System.out.println("   modification_time: " + new Date(status.getModificationTime()));
    System.out.println("   access_time: " + new Date(status.getAccessTime()));
    System.out.println("   owner: " + status.getOwner());
    System.out.println("   group: " + status.getGroup());
    System.out.println("   permission: " + status.getPermission());
    System.out.println();
}

From source file:ch.ethz.las.wikimining.mr.io.SequenceFileReader.java

public HashMap<E, V> read() throws IOException {
    if (!fs.exists(path)) {
        throw new IOException(path + " does not exist!");
    }//from w ww  .  j  a  va 2  s  .  c  o  m

    final FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
        if (status.isDirectory()) {
            continue;
        }
        if ("_SUCCESS".equals(status.getPath().getName())) {
            continue;
        }

        readContent(status);
    }

    return map;
}