Example usage for org.apache.hadoop.fs FileSystem globStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem globStatus.

Prototype

public FileStatus[] globStatus(Path pathPattern) throws IOException 

Source Link

Document

Return all the files that match filePattern and are not checksum files.

Usage

From source file:com.asakusafw.compiler.directio.DirectFileIoProcessorRunTest.java

License:Apache License

private List<Path> find(String target) throws IOException {
    FileSystem fs = FileSystem.get(tester.configuration());
    FileStatus[] list = fs.globStatus(getPath(target));
    if (list == null) {
        return Collections.emptyList();
    }/*from   ww w. jav a2 s  . co  m*/
    List<Path> results = new ArrayList<>();
    for (FileStatus file : list) {
        results.add(file.getPath());
    }
    return results;
}

---

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private List<Ex1> getList(Class<? extends FileExporterDescription> exporter) {
    try {/*from   ww w.j a  v a2  s  .  co m*/
        FileExporterDescription instance = exporter.newInstance();
        Path path = new Path(Location.fromPath(instance.getPathPrefix(), '/').toString());
        FileSystem fs = path.getFileSystem(tester.configuration());
        FileStatus[] statuses = fs.globStatus(path);
        List<Ex1> results = new ArrayList<>();
        for (FileStatus status : statuses) {
            try (SequenceFile.Reader reader = new SequenceFile.Reader(tester.configuration(),
                    SequenceFile.Reader.file(fs.makeQualified(status.getPath())))) {
                Ex1 model = new Ex1();
                while (reader.next(NullWritable.get(), model)) {
                    Ex1 copy = new Ex1();
                    copy.copyFrom(model);
                    results.add(copy);
                }
            }
        }
        Collections.sort(results, new Comparator<Ex1>() {
            @Override
            public int compare(Ex1 o1, Ex1 o2) {
                return o1.getSidOption().compareTo(o2.getSidOption());
            }
        });
        return results;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

---

From source file:com.asakusafw.compiler.util.tester.HadoopDriver.java

License:Apache License

private void copyFromHadoop(Location location, File targetDirectory) throws IOException {
    targetDirectory.mkdirs();/*w w w  . jav a  2 s.  c  om*/
    logger.info("copy {} to {}", location, targetDirectory);

    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(configuration);
    FileStatus[] list = fs.globStatus(path);
    if (list == null) {
        throw new IOException(
                MessageFormat.format("Failed to fs -get: source={0}, destination={1}", path, targetDirectory));
    }
    for (FileStatus status : list) {
        Path p = status.getPath();
        try {
            fs.copyToLocalFile(p, new Path(new File(targetDirectory, p.getName()).toURI()));
        } catch (IOException e) {
            throw new IOException(
                    MessageFormat.format("Failed to fs -get: source={0}, destination={1}", p, targetDirectory),
                    e);
        }
    }
}

---

From source file:com.asakusafw.dag.compiler.extension.internalio.testing.InternalIoTestHelper.java

License:Apache License

/**
 * Collects output data./*w w w.  j ava 2s  .com*/
 * @param <T> the data type
 * @param dataType the data type
 * @param path the input path
 * @param action the prepare action
 */
public <T extends Writable> void output(Class<T> dataType, String path, Action<List<T>, ?> action) {
    Configuration conf = new Configuration();
    Path p = new Path(locate(path).toURI());
    try {
        FileSystem fs = p.getFileSystem(conf);
        FileStatus[] stats = fs.globStatus(p);
        List<T> results = new ArrayList<>();
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = TemporaryStorage.openInput(conf, dataType, stat.getPath())) {
                while (true) {
                    T buf = dataType.newInstance();
                    if (in.readTo(buf) == false) {
                        break;
                    }
                    results.add(buf);
                }
            }
        }
        action.perform(results);
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

---

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalExporterRetriever.java

License:Apache License

@Override
public void truncate(InternalExporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;//from  w  w  w  .  ja v  a 2s . c o  m
    if (output == null) {
        LOG.warn(MessageFormat.format("skipped deleting output directory because it is a base directory: {0}",
                path));
        target = fs.makeQualified(path);
    } else {
        LOG.debug("output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    LOG.debug("deleting output target: {}", target); //$NON-NLS-1$
    try {
        FileStatus[] stats = fs.globStatus(path);
        for (FileStatus s : stats) {
            Path f = s.getPath();
            boolean deleted = fs.delete(f, true);
            LOG.debug("deleted output target (succeed={}): {}", deleted, f); //$NON-NLS-1$
        }
    } catch (IOException e) {
        LOG.debug("exception in truncate", e);
    }
}

---

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalImporterPreparator.java

License:Apache License

@Override
public void truncate(InternalImporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting input: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path target = fs.makeQualified(new Path(resolved));
    FileStatus[] stats = fs.globStatus(target);
    if (stats == null || stats.length == 0) {
        return;/*from  w w  w.  j ava  2  s .  c  om*/
    }
    for (FileStatus s : stats) {
        Path path = s.getPath();
        LOG.debug("deleting file: {}", path); //$NON-NLS-1$
        boolean succeed = fs.delete(path, true);
        LOG.debug("deleted file (succeed={}): {}", succeed, path); //$NON-NLS-1$
    }
    return;
}

---

From source file:com.asakusafw.m3bp.compiler.tester.externalio.TestIoTaskExecutor.java

License:Apache License

private <T extends Writable> void executeOutput(String name, Class<T> dataType, List<Path> paths)
        throws IOException {
    Action<Object, Exception> action = outputs.get(name);
    Invariants.requireNonNull(action, () -> MessageFormat.format("missing output: {0}", name));
    List<T> results = new ArrayList<>();
    for (Path pattern : paths) {
        FileSystem fs = pattern.getFileSystem(configuration);
        FileStatus[] stats = fs.globStatus(pattern);
        if (stats == null) {
            continue;
        }//from w  w w . j  av a  2s  .c om
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = new TemporaryFileInput<>(fs.open(stat.getPath()), 0)) {
                while (true) {
                    T instance = dataType.newInstance();
                    if (in.readTo(instance)) {
                        results.add(instance);
                    } else {
                        break;
                    }
                }
            } catch (Error | RuntimeException | IOException e) {
                throw e;
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        }
    }
    try {
        action.perform(results);
    } catch (Error | RuntimeException | IOException e) {
        throw e;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

---

From source file:com.asakusafw.operation.tools.hadoop.fs.Clean.java

License:Apache License

boolean remove(Path path, Context context) {
    LOG.info(MessageFormat.format("[OT-CLEAN-I01000] Start cleaning: {0}", path));
    FileSystem fs;
    try {/*from  www .ja  va2  s  .c om*/
        fs = FileSystem.get(path.toUri(), getConf());
    } catch (Exception e) {
        LOG.error(MessageFormat.format("[OT-CLEAN-E01001] Failed to connect to filesystem: {0}", path), e);
        context.setError();
        return false;
    }
    List<FileStatus> files;
    try {
        files = asList(fs.globStatus(path));
    } catch (Exception e) {
        LOG.error(MessageFormat.format("[OT-CLEAN-E01002] Failed to glob path pattern: {0}", path), e);
        context.setError();
        return false;
    }
    if (files.isEmpty()) {
        LOG.warn(MessageFormat.format("[OT-CLEAN-W01001] Target file is not found: {0}", path));
        context.setError();
        return false;
    }
    boolean removed = true;
    long start = System.currentTimeMillis();
    for (FileStatus file : files) {
        removed &= remove(fs, file, context);
    }
    long end = System.currentTimeMillis();
    LOG.info(MessageFormat.format("[OT-CLEAN-I01999] Finish cleaning: {0} (all-removed={1}, elapsed={2}ms)",
            path, removed, end - start));
    return removed;
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static List<FileStatus> globStep(FileSystem fs, List<FileStatus> current, List<Path> expressions)
        throws IOException {
    assert fs != null;
    assert current != null;
    assert expressions != null;
    Set<Path> paths = new HashSet<>();
    List<FileStatus> results = new ArrayList<>();
    for (FileStatus status : current) {
        if (FileSystemCompatibility.isDirectory(status) == false) {
            continue;
        }/* w w w . ja  v  a  2  s. c  o m*/
        for (Path expression : expressions) {
            Path path = new Path(status.getPath(), expression);
            FileStatus[] expanded = fs.globStatus(path);
            if (expanded != null) {
                for (FileStatus s : expanded) {
                    Path p = s.getPath();
                    if (paths.contains(p) == false) {
                        paths.add(p);
                        results.add(s);
                    }
                }
            }
        }
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormat.java

License:Apache License

private List<InputSplit> getSplits(Configuration configuration, List<Path> paths) throws IOException {
    long splitSize = configuration.getLong(KEY_DEFAULT_SPLIT_SIZE, DEFAULT_SPLIT_SIZE);
    List<InputSplit> results = new ArrayList<>();
    for (Path path : paths) {
        FileSystem fs = path.getFileSystem(configuration);
        FileStatus[] statuses = fs.globStatus(path);
        if (statuses == null) {
            continue;
        }//from w  w w.j ava 2s.c  o m
        for (FileStatus status : statuses) {
            BlockMap blockMap = BlockMap.create(status.getPath().toString(), status.getLen(),
                    BlockMap.computeBlocks(fs, status), false);
            results.addAll(computeSplits(status.getPath(), blockMap, splitSize));
        }
    }
    return results;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »