Example usage for org.apache.hadoop.fs FileStatus getPath

List of usage examples for org.apache.hadoop.fs FileStatus getPath

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileStatus
  6. getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath()

Source Link

Usage

From source file:com.asakusafw.compiler.directio.DirectFileIoProcessorRunTest.java

License:Apache License

private List<Path> find(String target) throws IOException {
    FileSystem fs = FileSystem.get(tester.configuration());
    FileStatus[] list = fs.globStatus(getPath(target));
    if (list == null) {
        return Collections.emptyList();
    }//from  w w w .  j  ava  2 s  .  c  o  m
    List<Path> results = new ArrayList<>();
    for (FileStatus file : list) {
        results.add(file.getPath());
    }
    return results;
}

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private List<Ex1> getList(Class<? extends FileExporterDescription> exporter) {
    try {//ww w .j ava  2s  .  c o m
        FileExporterDescription instance = exporter.newInstance();
        Path path = new Path(Location.fromPath(instance.getPathPrefix(), '/').toString());
        FileSystem fs = path.getFileSystem(tester.configuration());
        FileStatus[] statuses = fs.globStatus(path);
        List<Ex1> results = new ArrayList<>();
        for (FileStatus status : statuses) {
            try (SequenceFile.Reader reader = new SequenceFile.Reader(tester.configuration(),
                    SequenceFile.Reader.file(fs.makeQualified(status.getPath())))) {
                Ex1 model = new Ex1();
                while (reader.next(NullWritable.get(), model)) {
                    Ex1 copy = new Ex1();
                    copy.copyFrom(model);
                    results.add(copy);
                }
            }
        }
        Collections.sort(results, new Comparator<Ex1>() {
            @Override
            public int compare(Ex1 o1, Ex1 o2) {
                return o1.getSidOption().compareTo(o2.getSidOption());
            }
        });
        return results;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

From source file:com.asakusafw.compiler.util.tester.HadoopDriver.java

License:Apache License

private void copyFromHadoop(Location location, File targetDirectory) throws IOException {
    targetDirectory.mkdirs();//from w  w  w . ja v  a  2s.c o m
    logger.info("copy {} to {}", location, targetDirectory);

    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(configuration);
    FileStatus[] list = fs.globStatus(path);
    if (list == null) {
        throw new IOException(
                MessageFormat.format("Failed to fs -get: source={0}, destination={1}", path, targetDirectory));
    }
    for (FileStatus status : list) {
        Path p = status.getPath();
        try {
            fs.copyToLocalFile(p, new Path(new File(targetDirectory, p.getName()).toURI()));
        } catch (IOException e) {
            throw new IOException(
                    MessageFormat.format("Failed to fs -get: source={0}, destination={1}", p, targetDirectory),
                    e);
        }
    }
}

From source file:com.asakusafw.dag.compiler.extension.internalio.testing.InternalIoTestHelper.java

License:Apache License

/**
 * Collects output data./* ww w.  ja v  a 2  s  .c om*/
 * @param <T> the data type
 * @param dataType the data type
 * @param path the input path
 * @param action the prepare action
 */
public <T extends Writable> void output(Class<T> dataType, String path, Action<List<T>, ?> action) {
    Configuration conf = new Configuration();
    Path p = new Path(locate(path).toURI());
    try {
        FileSystem fs = p.getFileSystem(conf);
        FileStatus[] stats = fs.globStatus(p);
        List<T> results = new ArrayList<>();
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = TemporaryStorage.openInput(conf, dataType, stat.getPath())) {
                while (true) {
                    T buf = dataType.newInstance();
                    if (in.readTo(buf) == false) {
                        break;
                    }
                    results.add(buf);
                }
            }
        }
        action.perform(results);
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

From source file:com.asakusafw.dag.runtime.internalio.InternalInputAdapter.java

License:Apache License

private <T extends Writable> void resolve(List<Path> paths, Class<T> type, Consumer<TaskInfo> sink)
        throws IOException {
    FileSystem fs = FileSystem.get(configuration);
    Supplier<? extends T> supplier = () -> objectFactory.newInstance(type);
    List<FileStatus> stats = new ArrayList<>();
    for (Path path : paths) {
        List<FileStatus> s = TemporaryStorage.listStatus(configuration, path);
        stats.addAll(s);/*from   w  w  w  . j  a  va 2  s . c o m*/
    }
    for (FileStatus stat : stats) {
        Path p = stat.getPath();
        File local = null;
        URI uri = p.toUri();
        String scheme = uri.getScheme();
        if (scheme != null && scheme.equals("file")) { //$NON-NLS-1$
            local = new File(uri);
        }
        long length = stat.getLen();
        if (length == 0) {
            continue;
        }
        int blocks = (int) ((length + TemporaryFile.BLOCK_SIZE - 1) / TemporaryFile.BLOCK_SIZE);
        for (int i = 0; i < blocks; i++) {
            if (local == null) {
                sink.accept(new HadoopInternalInputTaskInfo<>(fs, p, i, 1, supplier));
            } else {
                sink.accept(new LocalInternalInputTaskInfo<>(local, i, 1, supplier));
            }
        }
    }
}

From source file:com.asakusafw.directio.hive.orc.AbstractOrcFileFormat.java

License:Apache License

@Override
public List<DirectInputFragment> computeInputFragments(InputContext context)
        throws IOException, InterruptedException {
    // TODO parallel?
    List<DirectInputFragment> results = new ArrayList<>();
    for (FileStatus status : context.getInputFiles()) {
        if (LOG.isInfoEnabled()) {
            LOG.info(MessageFormat.format(Messages.getString("AbstractOrcFileFormat.infoLoadMetadata"), //$NON-NLS-1$
                    context.getDataType().getSimpleName(), status.getPath()));
        }/*from www .  j a v a 2 s .c o  m*/
        Reader orc = OrcFile.createReader(context.getFileSystem(), status.getPath());
        if (LOG.isInfoEnabled()) {
            LOG.info(MessageFormat.format(Messages.getString("AbstractOrcFileFormat.infoAnalyzeMetadata"), //$NON-NLS-1$
                    context.getDataType().getSimpleName(), status.getPath(), orc.getNumberOfRows(),
                    orc.getRawDataSize()));
        }
        BlockMap blockMap = BlockMap.create(status.getPath().toString(), status.getLen(),
                BlockMap.computeBlocks(context.getFileSystem(), status), false);
        // TODO configurable split
        for (StripeInformation stripe : orc.getStripes()) {
            long begin = stripe.getOffset();
            long end = begin + stripe.getLength();
            DirectInputFragment fragment = blockMap.get(begin, end);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format(
                        "Detect ORCFile stripe: path={0}, rows={1}, range={2}+{3}, allocation={4}", //$NON-NLS-1$
                        fragment.getPath(), stripe.getNumberOfRows(), fragment.getOffset(), fragment.getSize(),
                        fragment.getOwnerNodeNames()));
            }
            results.add(fragment);
        }
    }
    return results;
}

From source file:com.asakusafw.directio.hive.parquet.AbstractParquetFileFormat.java

License:Apache License

@Override
public List<DirectInputFragment> computeInputFragments(InputContext context)
        throws IOException, InterruptedException {
    List<DirectInputFragment> results = new ArrayList<>();
    List<FileStatus> files = new ArrayList<>(context.getInputFiles());
    Map<Path, FileStatus> pathMap = new HashMap<>();
    for (FileStatus status : files) {
        pathMap.put(status.getPath(), status);
    }//from  ww w  . j a v  a2 s .  co  m
    if (LOG.isInfoEnabled()) {
        LOG.info(MessageFormat.format(Messages.getString("AbstractParquetFileFormat.infoLoadMetadata"), //$NON-NLS-1$
                context.getDataType().getSimpleName(), files.size()));
    }
    List<Footer> footers = ParquetFileReader.readAllFootersInParallel(getConf(), files);
    for (Footer footer : footers) {
        Path path = footer.getFile();
        FileStatus status = pathMap.get(path);
        if (status == null) {
            // may not occur
            status = context.getFileSystem().getFileStatus(path);
        }
        if (LOG.isInfoEnabled()) {
            LOG.info(MessageFormat.format(Messages.getString("AbstractParquetFileFormat.infoAnalyzeMetadata"), //$NON-NLS-1$
                    context.getDataType().getSimpleName(), status.getPath()));
        }
        BlockMap blockMap = BlockMap.create(status.getPath().toString(), status.getLen(),
                BlockMap.computeBlocks(context.getFileSystem(), status), false);
        for (BlockMetaData block : footer.getParquetMetadata().getBlocks()) {
            if (block.getColumns().isEmpty()) {
                continue;
            }
            long begin = Long.MAX_VALUE;
            long end = -1L;
            for (ColumnChunkMetaData column : block.getColumns()) {
                long offset = column.getFirstDataPageOffset();
                long size = column.getTotalSize();
                begin = Math.min(begin, offset);
                end = Math.max(end, offset + size);
            }
            assert begin >= 0;
            assert end >= 0;
            DirectInputFragment fragment = blockMap.get(begin, end);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Detect Parquet file block: " //$NON-NLS-1$
                        + "path={0}, rows={1}, range={2}+{3}, allocation={4}", //$NON-NLS-1$
                        status.getPath(), block.getRowCount(), begin, end - begin,
                        fragment.getOwnerNodeNames()));
            }
            results.add(fragment);
        }
    }
    return results;
}

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalExporterRetriever.java

License:Apache License

@Override
public void truncate(InternalExporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;/*from w ww.j a v  a  2 s  .c o m*/
    if (output == null) {
        LOG.warn(MessageFormat.format("skipped deleting output directory because it is a base directory: {0}",
                path));
        target = fs.makeQualified(path);
    } else {
        LOG.debug("output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    LOG.debug("deleting output target: {}", target); //$NON-NLS-1$
    try {
        FileStatus[] stats = fs.globStatus(path);
        for (FileStatus s : stats) {
            Path f = s.getPath();
            boolean deleted = fs.delete(f, true);
            LOG.debug("deleted output target (succeed={}): {}", deleted, f); //$NON-NLS-1$
        }
    } catch (IOException e) {
        LOG.debug("exception in truncate", e);
    }
}

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalImporterPreparator.java

License:Apache License

@Override
public void truncate(InternalImporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting input: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path target = fs.makeQualified(new Path(resolved));
    FileStatus[] stats = fs.globStatus(target);
    if (stats == null || stats.length == 0) {
        return;/*from   w w  w  .  ja  v a 2  s.co  m*/
    }
    for (FileStatus s : stats) {
        Path path = s.getPath();
        LOG.debug("deleting file: {}", path); //$NON-NLS-1$
        boolean succeed = fs.delete(path, true);
        LOG.debug("deleted file (succeed={}): {}", succeed, path); //$NON-NLS-1$
    }
    return;
}

From source file:com.asakusafw.m3bp.compiler.tester.externalio.TestIoTaskExecutor.java

License:Apache License

private <T extends Writable> void executeOutput(String name, Class<T> dataType, List<Path> paths)
        throws IOException {
    Action<Object, Exception> action = outputs.get(name);
    Invariants.requireNonNull(action, () -> MessageFormat.format("missing output: {0}", name));
    List<T> results = new ArrayList<>();
    for (Path pattern : paths) {
        FileSystem fs = pattern.getFileSystem(configuration);
        FileStatus[] stats = fs.globStatus(pattern);
        if (stats == null) {
            continue;
        }/*ww  w . j a va  2s . c om*/
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = new TemporaryFileInput<>(fs.open(stat.getPath()), 0)) {
                while (true) {
                    T instance = dataType.newInstance();
                    if (in.readTo(instance)) {
                        results.add(instance);
                    } else {
                        break;
                    }
                }
            } catch (Error | RuntimeException | IOException e) {
                throw e;
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        }
    }
    try {
        action.perform(results);
    } catch (Error | RuntimeException | IOException e) {
        throw e;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}