Example usage for org.apache.hadoop.fs FileSystem globStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem globStatus.

Prototype

public FileStatus[] globStatus(Path pathPattern) throws IOException 

Source Link

Document

Return all the files that match filePattern and are not checksum files.

Usage

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Resolves the raw path pattern into the concrete file status list.
 * @param conf current configuration//from  w  w w  .  j  a v a 2  s.c  om
 * @param pathPattern path pattern which describes temporary storage
 * @return the resolved file status
 * @throws IOException if failed to resolve path pattern
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.7.1
 */
public static List<FileStatus> listStatus(Configuration conf, Path pathPattern) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (pathPattern == null) {
        throw new IllegalArgumentException("pathPattern must not be null"); //$NON-NLS-1$
    }
    FileSystem fs = pathPattern.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Listing temporary input: {0} (fs={1})", //$NON-NLS-1$
                pathPattern, fs.getUri()));
    }
    FileStatus[] statusList = fs.globStatus(pathPattern);
    if (statusList == null || statusList.length == 0) {
        return Collections.emptyList();
    }
    return Arrays.asList(statusList);
}

---

From source file:com.asakusafw.testdriver.FlowPartTestDriver.java

License:Apache License

private void loadResult(String tablename, String excelFileName) throws IOException {
    Configuration conf = ConfigurationFactory.getDefault().newInstance();
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] status = fs.globStatus(new Path(computeOutputPath(fs, excelFileName)));
    Path[] listedPaths = FileUtil.stat2Paths(status);
    for (Path path : listedPaths) {
        if (isSystemFile(path)) {
            continue;
        }/*from w  w w .j  ava2s . com*/
        LOG.info("????:Path=" + path);
        testUtils.loadFromTemporary(tablename, conf, path);
    }
}

---

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryInputPreparator.java

License:Apache License

static void delete(FileSystem fs, Path target) throws IOException {
    FileStatus[] stats = fs.globStatus(target);
    if (stats == null || stats.length == 0) {
        return;// www.  j a v  a2s. c o m
    }
    for (FileStatus s : stats) {
        Path path = s.getPath();
        LOG.debug("deleting file: {}", path); //$NON-NLS-1$
        boolean succeed = fs.delete(path, true);
        LOG.debug("deleted file (succeed={}): {}", succeed, path); //$NON-NLS-1$
    }
}

---

From source file:com.asakusafw.testdriver.testing.moderator.MockExporterRetriever.java

License:Apache License

@Override
public void truncate(MockExporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting output directory: {}", description); //$NON-NLS-1$
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    Path path = new Path(description.getGlob());
    try {//from ww  w .j a  v a 2  s.  co m
        FileStatus[] stats = fs.globStatus(path);
        for (FileStatus s : stats) {
            fs.delete(s.getPath(), false);
        }
    } catch (IOException e) {
        LOG.debug("exception in truncate", e);
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopDelete.java

License:Apache License

void doDelete(List<Path> paths, FileList.Writer drain) throws IOException {
    assert paths != null;
    assert drain != null;
    FileSystem fs = FileSystem.get(conf);
    for (Path path : paths) {
        WGLOG.info("I22003", fs.getUri(), path);
        FileStatus[] results = fs.globStatus(path);
        if (results == null) {
            continue;
        }/*from  ww  w .  j ava2  s . c o m*/
        for (FileStatus status : results) {
            doDelete(fs, status, drain);
        }
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGet.java

License:Apache License

void fetch(FileSystem fs, List<Path> paths, BlockingQueue<Pair> queue)
        throws IOException, InterruptedException {
    assert fs != null;
    assert paths != null;
    assert queue != null;
    for (Path path : paths) {
        boolean found = false;
        WGLOG.info("I20003", fs.getUri(), path);
        FileStatus[] results = fs.globStatus(path);
        if (results != null) {
            for (FileStatus status : results) {
                if (FileSystemCompatibility.isDirectory(status)) {
                    continue;
                }/*from  ww w .ja va2s  .c o m*/
                found = true;
                InputStream in = getInput(fs, status);
                boolean succeed = false;
                try {
                    queue.put(new Pair(in, status));
                    succeed = true;
                } finally {
                    if (succeed == false) {
                        in.close();
                    }
                }
            }
        }
        if (found == false && RuntimeContext.get().isSimulation() == false) {
            throw new FileNotFoundException(paths.toString());
        }
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.temporary.FileSystemModelInputProvider.java

License:Apache License

/**
 * Creates a new instance.//from  www  .j  a v a 2 s  .  co  m
 * @param configuration the configuration
 * @param fileSystem target file system
 * @param paths source paths
 * @param dataModelClass target data model class
 * @throws IOException if failed to resolve paths
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public FileSystemModelInputProvider(final Configuration configuration, final FileSystem fileSystem,
        final Iterable<Path> paths, final Class<T> dataModelClass) throws IOException {
    if (configuration == null) {
        throw new IllegalArgumentException("configuration must not be null"); //$NON-NLS-1$
    }
    if (fileSystem == null) {
        throw new IllegalArgumentException("fileSystem must not be null"); //$NON-NLS-1$
    }
    if (paths == null) {
        throw new IllegalArgumentException("paths must not be null"); //$NON-NLS-1$
    }
    if (dataModelClass == null) {
        throw new IllegalArgumentException("dataModelClass must not be null"); //$NON-NLS-1$
    }
    this.fileSystem = fileSystem;
    this.queue = new SynchronousQueue<>();
    this.executor = Executors.newFixedThreadPool(1, new ThreadFactory() {
        @Override
        public Thread newThread(Runnable r) {
            Thread t = new Thread(r, "HadoopFileCollector");
            t.setDaemon(true);
            return t;
        }
    });
    this.fetcher = this.executor.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
            for (Path path : paths) {
                WGLOG.info("I09001", fileSystem.getUri(), paths);
                FileStatus[] statusList = fileSystem.globStatus(path);
                if (statusList == null || statusList.length == 0) {
                    throw new FileNotFoundException(MessageFormat.format("File is not found in {1} (fs={0})",
                            fileSystem.getUri(), paths));
                }
                for (FileStatus status : statusList) {
                    WGLOG.info("I09002", fileSystem.getUri(), status.getPath(), status.getLen());
                    ModelInput<T> input = TemporaryStorage.openInput(configuration, dataModelClass,
                            status.getPath());
                    boolean succeed = false;
                    try {
                        queue.put(new Entry<>(status, input));
                        succeed = true;
                    } finally {
                        if (succeed == false) {
                            input.close();
                        }
                    }
                }
            }
            queue.put(Entry.<T>eof());
            return null;
        }
    });
}

---

From source file:com.blackberry.logdriver.admin.HFind.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    final long startTime = System.currentTimeMillis();

    int i = 0;// www .j a va  2 s  .c o  m
    while (i < args.length) {
        if (args[i].startsWith("-")) {
            break;
        }

        Path path = new Path(args[i]);
        FileSystem fs = path.getFileSystem(getConf());
        FileStatus[] fileStatuses = fs.globStatus(path);
        if (fileStatuses != null) {
            for (FileStatus fileStatus : fileStatuses) {
                paths.add(fileStatus.getPath());
                fileStatusCache.put(fileStatus.getPath(), fileStatus);
            }
        }

        i++;
    }

    while (i < args.length) {
        // -print action
        if ("-print".equals(args[i])) {
            actions.add(new FileStatusFilter() {
                @Override
                public boolean accept(FileStatus fileStatus) {
                    System.out.println(fileStatus.getPath());
                    return true;
                }
            });
        }

        // -delete action
        if ("-delete".equals(args[i])) {
            actions.add(new FileStatusFilter() {
                @SuppressWarnings("deprecation")
                @Override
                public boolean accept(FileStatus fileStatus) {
                    try {
                        FileSystem fs = fileStatus.getPath().getFileSystem(getConf());
                        if (!fileStatus.isDir() || fs.listStatus(fileStatus.getPath()).length == 0) {
                            return fs.delete(fileStatus.getPath(), true);
                        }
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    return false;
                }
            });
        }

        // -atime test
        else if ("-atime".equals(args[i])) {
            i++;
            if (i >= args.length) {
                System.err.println("Missing arguement for -atime");
                System.exit(1);
            }

            String t = args[i];
            if (t.charAt(0) == '+') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) > time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else if (t.charAt(0) == '-') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) < time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else {
                final long time = Long.parseLong(t);
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (24 * 60 * 60 * 1000) == time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            }
        }

        // -mtime test
        else if ("-mtime".equals(args[i])) {
            i++;
            if (i >= args.length) {
                System.err.println("Missing arguement for -mtime");
                System.exit(1);
            }

            String t = args[i];
            if (t.charAt(0) == '+') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) > time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else if (t.charAt(0) == '-') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) < time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else {
                final long time = Long.parseLong(t);
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (24 * 60 * 60 * 1000) == time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            }
        }

        // -amin test
        else if ("-amin".equals(args[i])) {
            i++;
            if (i >= args.length) {
                System.err.println("Missing arguement for -amin");
                System.exit(1);
            }

            String t = args[i];
            if (t.charAt(0) == '+') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) > time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else if (t.charAt(0) == '-') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) < time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else {
                final long time = Long.parseLong(t);
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getAccessTime()) / (60 * 1000) == time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            }
        }

        // -mmin test
        else if ("-mmin".equals(args[i])) {
            i++;
            if (i >= args.length) {
                System.err.println("Missing arguement for -mmin");
                System.exit(1);
            }

            String t = args[i];
            if (t.charAt(0) == '+') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) > time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else if (t.charAt(0) == '-') {
                final long time = Long.parseLong(t.substring(1));
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) < time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            } else {
                final long time = Long.parseLong(t);
                tests.add(new FileStatusFilter() {
                    @Override
                    public boolean accept(FileStatus fileStatus) {
                        if ((startTime - fileStatus.getModificationTime()) / (60 * 1000) == time) {
                            return true;
                        } else {
                            return false;
                        }
                    }
                });
            }
        }

        // -regex test
        else if ("-regex".equals(args[i])) {
            i++;
            if (i >= args.length) {
                System.err.println("Missing arguement for -regex");
                System.exit(1);
            }

            final Pattern p = Pattern.compile(args[i]);
            tests.add(new FileStatusFilter() {
                @Override
                public boolean accept(FileStatus fileStatus) {
                    if (p.matcher(fileStatus.getPath().toString()).matches()) {
                        return true;
                    } else {
                        return false;
                    }
                }
            });
        }

        i++;
    }

    if (actions.size() == 0) {
        actions.add(new FileStatusFilter() {
            @Override
            public boolean accept(FileStatus fileStatus) {
                System.out.println(fileStatus.getPath());
                return true;
            }
        });
    }

    search();

    return 0;
}

---

From source file:com.blackberry.logdriver.LockedFs.java

License:Apache License

@SuppressWarnings("deprecation")
public void move(Configuration conf, String[] from, String to) throws IOException {
    FileSystem fs = FileSystem.get(conf);

    List<FileStatus> fromList = new ArrayList<FileStatus>();
    for (String s : from) {
        FileStatus[] statuses = fs.globStatus(new Path(s));
        if (statuses == null) {
            continue;
        }/*from ww  w.  ja  v  a  2s.  co  m*/
        for (FileStatus status : statuses) {
            fromList.add(status);
        }
    }

    Path toPath = new Path(to);
    Boolean toExists = fs.exists(toPath);
    FileStatus toFileStatus = null;
    if (toExists) {
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there is no from, that's a problem.
    if (fromList.isEmpty()) {
        throw new IOException("No input files found");
    }

    // If the to exists, and is a file, that's a problem too.
    if (toExists && !toFileStatus.isDir()) {
        throw new IOException("Destination file exists:" + to);
    }

    // If the destination exists, and is a directory, then ensure that none of
    // the from list names will clash with existing contents of the directory.
    if (toExists && toFileStatus.isDir()) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            if (fs.exists(new Path(toPath, name))) {
                throw new IOException("Destination file exists:" + to + "/" + name);
            }
        }
    }

    // If the destination doesn't exist, but it ends with a slash, then create
    // it as a directory.
    if (!toExists && to.endsWith("/")) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
        toExists = true;
    }

    // If the destination doesn't exist, and there is more than one 'from', then
    // create a directory.
    if (!toExists && fromList.size() > 1) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there was only one from, then just rename it to to
    if (fromList.size() == 1) {
        fs.mkdirs(toPath.getParent());
        fs.rename(fromList.get(0).getPath(), toPath);
    }

    // If there was more than one from, then for each file in the from list,
    // move it to the to directory.
    if (fromList.size() > 1) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            fs.rename(fromStatus.getPath(), new Path(toPath, name));
        }
    }
}

---

From source file:com.blackberry.logdriver.util.Cat.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf(); // Configuration processed by ToolRunner
    // If run by Oozie, then load the Oozie conf too
    if (System.getProperty("oozie.action.conf.xml") != null) {
        conf.addResource(new URL("file://" + System.getProperty("oozie.action.conf.xml")));
    }/* w  ww.j  a  v  a  2s.  c o m*/

    FileSystem fs = FileSystem.get(conf);

    // The command line options
    List<Path> paths = new ArrayList<Path>();
    Path outputDir = null;

    // Load input files from the command line
    if (args.length < 2) {
        System.out.println("usage: [genericOptions] input [input ...] output");
        System.exit(1);
    }

    // Get the files we need from the command line.
    for (int i = 0; i < args.length - 1; i++) {
        for (FileStatus f : fs.globStatus(new Path(args[i]))) {
            paths.add(f.getPath());
        }
    }
    outputDir = new Path(args[args.length - 1]);

    @SuppressWarnings("deprecation")
    Job job = new Job(conf);
    Configuration jobConf = job.getConfiguration();

    job.setJarByClass(Cat.class);
    jobConf.setIfUnset("mapred.job.name", "Cat Files");

    // To propagate credentials within Oozie
    if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
        jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
    }

    // Good output separators include things that are unsupported by XML. So we
    // just send the byte value of the character through. The restriction here
    // is that it can't be more than 1 byte when UTF-8 encoded, since it will be
    // read by Pig which only deals with single byte separators.
    {
        String outputSeparator = jobConf.get("logdriver.output.field.separator", DEFAULT_OUTPUT_SEPARATOR);
        byte[] bytes = outputSeparator.getBytes(UTF_8);
        if (bytes.length != 1) {
            LOG.error("The output separator must be a single byte in UTF-8.");
            return 1;
        }
        jobConf.set("logdriver.output.field.separator", Byte.toString(bytes[0]));
    }

    job.setInputFormatClass(BoomInputFormat.class);
    job.setMapperClass(CatMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, outputDir);

    for (Path path : paths) {
        BoomInputFormat.addInputPath(job, path);
    }

    // Run the job.
    if (conf.getBoolean("job.wait", DEFAULT_WAIT_JOB)) {
        return job.waitForCompletion(true) ? 0 : 1;
    } else {
        job.submit();
        return 0;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »