Example usage for org.apache.hadoop.fs Path toString

List of usage examples for org.apache.hadoop.fs Path toString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString()

Source Link

Usage

From source file:com.asakusafw.lang.compiler.mapreduce.CopyStageEmitterTest.java

License:Apache License

/**
 * simple case./*from   w w w  . j  av  a 2  s  .  com*/
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    FileEditor.put(new File(folder.getRoot(), "input/test.txt"), "Hello, world!");
    Path root = new Path(folder.getRoot().toURI());
    Path base = new Path(root, "output");
    ClassDescription client = new ClassDescription("com.example.StageClient");
    CopyStageInfo info = new CopyStageInfo(new StageInfo("simple", "simple", "simple"),
            Arrays.asList(new CopyStageInfo.Operation("out",
                    new SourceInfo(new Path(root, "input/*.txt").toString(), classOf(Text.class),
                            classOf(TextInputFormat.class), Collections.emptyMap()),
                    classOf(TextOutputFormat.class), Collections.emptyMap())),
            base.toString());
    CopyStageEmitter.emit(client, info, javac);
    int status = MapReduceRunner.execute(new Configuration(), client, "testing", Collections.emptyMap(),
            javac.compile());
    assertThat("exit status code", status, is(0));

    assertThat(collect(MapReduceUtil.getStageOutputPath(base.toString(), "out")), contains("Hello, world!"));
}

From source file:com.asakusafw.lang.compiler.mapreduce.CopyStageEmitterTest.java

License:Apache License

/**
 * multiple files.// www .  ja v a2  s.com
 * @throws Exception if failed
 */
@Test
public void multiple() throws Exception {
    FileEditor.put(new File(folder.getRoot(), "input/test0.txt"), "Hello0");
    FileEditor.put(new File(folder.getRoot(), "input/test1.txt"), "Hello1");
    FileEditor.put(new File(folder.getRoot(), "input/test2.txt"), "Hello2");

    Path root = new Path(folder.getRoot().toURI());
    Path base = new Path(root, "output");
    ClassDescription client = new ClassDescription("com.example.StageClient");
    CopyStageInfo info = new CopyStageInfo(new StageInfo("simple", "simple", "simple"), Arrays
            .asList(new CopyStageInfo.Operation[] {
                    new CopyStageInfo.Operation(
                            "out0",
                            new SourceInfo(
                                    new Path(root, "input/test0.txt").toString(), classOf(Text.class),
                                    classOf(TextInputFormat.class), Collections.emptyMap()),
                            classOf(TextOutputFormat.class), Collections.emptyMap()),
                    new CopyStageInfo.Operation("out1",
                            new SourceInfo(new Path(root, "input/test1.txt").toString(), classOf(Text.class),
                                    classOf(TextInputFormat.class), Collections.emptyMap()),
                            classOf(TextOutputFormat.class), Collections.emptyMap()),
                    new CopyStageInfo.Operation("out2",
                            new SourceInfo(new Path(root, "input/test2.txt").toString(), classOf(Text.class),
                                    classOf(TextInputFormat.class), Collections.emptyMap()),
                            classOf(TextOutputFormat.class), Collections.emptyMap()), }),
            base.toString());
    CopyStageEmitter.emit(client, info, javac);
    int status = MapReduceRunner.execute(new Configuration(), client, "testing", Collections.emptyMap(),
            javac.compile());
    assertThat("exit status code", status, is(0));

    assertThat(collect(MapReduceUtil.getStageOutputPath(base.toString(), "out0")), contains("Hello0"));
    assertThat(collect(MapReduceUtil.getStageOutputPath(base.toString(), "out1")), contains("Hello1"));
    assertThat(collect(MapReduceUtil.getStageOutputPath(base.toString(), "out2")), contains("Hello2"));
}

From source file:com.asakusafw.lang.compiler.mapreduce.MapReduceStageEmitterTest.java

License:Apache License

/**
 * simple case./* w  ww .j a  va2s . c  om*/
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    FileEditor.put(new File(folder.getRoot(), "input/test.txt"), "Hello, world!");
    Path root = new Path(folder.getRoot().toURI());
    Path base = new Path(root, "output");
    ClassDescription client = new ClassDescription("com.example.StageClient");
    MapReduceStageInfo info = new MapReduceStageInfo(new StageInfo("simple", "simple", "simple"),
            Arrays.asList(new MapReduceStageInfo.Input(new Path(root, "input/*.txt").toString(),
                    classOf(Text.class), classOf(TextInputFormat.class), classOf(SimpleMapper.class),
                    Collections.emptyMap())),
            Arrays.asList(new MapReduceStageInfo.Output("out", classOf(NullWritable.class), classOf(Text.class),
                    classOf(TextOutputFormat.class), Collections.emptyMap())),
            Collections.emptyList(), base.toString());
    MapReduceStageEmitter.emit(client, info, javac);
    int status = MapReduceRunner.execute(new Configuration(), client, "testing", Collections.emptyMap(),
            javac.compile());
    assertThat("exit status code", status, is(0));
    assertThat(collect("output"), contains("Hello, world!"));
}

From source file:com.asakusafw.lang.compiler.mapreduce.MapReduceStageEmitterTest.java

License:Apache License

/**
 * w/ reduce./*from   w  w  w  .j a v a  2 s  .  c  o  m*/
 * @throws Exception if failed
 */
@Test
public void reduce() throws Exception {
    FileEditor.put(new File(folder.getRoot(), "input/test.txt"), "Hello, world!");
    Path root = new Path(folder.getRoot().toURI());
    Path base = new Path(root, "output");
    ClassDescription client = new ClassDescription("com.example.StageClient");
    MapReduceStageInfo info = new MapReduceStageInfo(new StageInfo("simple", "simple", "simple"),
            Arrays.asList(
                    new MapReduceStageInfo.Input(new Path(root, "input/*.txt").toString(), classOf(Text.class),
                            classOf(TextInputFormat.class), classOf(Mapper.class), Collections.emptyMap())),
            Arrays.asList(new MapReduceStageInfo.Output("out", classOf(NullWritable.class), classOf(Text.class),
                    classOf(TextOutputFormat.class), Collections.emptyMap())),
            Collections.emptyList(),
            new MapReduceStageInfo.Shuffle(classOf(LongWritable.class), classOf(Text.class),
                    classOf(HashPartitioner.class), null, classOf(LongWritable.Comparator.class),
                    classOf(LongWritable.Comparator.class), classOf(SimpleReducer.class)),
            base.toString());
    MapReduceStageEmitter.emit(client, info, javac);
    int status = MapReduceRunner.execute(new Configuration(), client, "testing", Collections.emptyMap(),
            javac.compile());
    assertThat("exit status code", status, is(0));
    assertThat(collect("output"), contains("Hello, world!"));
}

From source file:com.asakusafw.lang.compiler.mapreduce.MapReduceStageEmitterTest.java

License:Apache License

/**
 * w/ resources./*from   w  ww  . j a  v a  2  s. co  m*/
 * @throws Exception if failed
 */
@Test
public void resource() throws Exception {
    FileEditor.put(new File(folder.getRoot(), "input/test.txt"), "Hello, input!");
    FileEditor.put(new File(folder.getRoot(), "resource/test.txt"), "Hello, resource!");
    Path root = new Path(folder.getRoot().toURI());
    Path base = new Path(root, "output");
    ClassDescription client = new ClassDescription("com.example.StageClient");
    MapReduceStageInfo info = new MapReduceStageInfo(new StageInfo("simple", "simple", "simple"),
            Arrays.asList(new MapReduceStageInfo.Input(new Path(root, "input/*.txt").toString(),
                    classOf(Text.class), classOf(TextInputFormat.class), classOf(ResourceMapper.class),
                    Collections.emptyMap())),
            Arrays.asList(new MapReduceStageInfo.Output("out", classOf(NullWritable.class), classOf(Text.class),
                    classOf(TextOutputFormat.class), Collections.emptyMap())),
            Arrays.asList(
                    new MapReduceStageInfo.Resource(new Path(root, "resource/*.txt").toString(), "resource")),
            base.toString());
    MapReduceStageEmitter.emit(client, info, javac);
    int status = MapReduceRunner.execute(new Configuration(), client, "testing", Collections.emptyMap(),
            javac.compile());
    assertThat("exit status code", status, is(0));
    assertThat(collect("output"), contains("Hello, resource!"));
}

From source file:com.asakusafw.lang.compiler.mapreduce.MapReduceUtil.java

License:Apache License

/**
 * Returns the actual path of the target stage output.
 * @param basePath the stage output base path
 * @param outputName the stage output name (see {@link #quoteOutputName(String)})
 * @return the processed path/*from w w  w.  j  a  va2  s .c  om*/
 */
public static String getStageOutputPath(String basePath, String outputName) {
    String fileName = String.format("%s-*", outputName); //$NON-NLS-1$
    Path path = new Path(basePath, fileName);
    return path.toString();
}

From source file:com.asakusafw.lang.compiler.mapreduce.testing.mock.DirectIoContext.java

License:Apache License

/**
 * Adds Direct I/O settings for the configuration.
 * @param conf the target configuration//from  w w  w. ja v  a2s  .co m
 * @return the configured object
 */
public Configuration configure(Configuration conf) {
    Path system = new Path(new File(getRoot(), "system").toURI()); //$NON-NLS-1$
    conf.set("com.asakusafw.output.system.dir", system.toString()); //$NON-NLS-1$
    conf.set("com.asakusafw.directio.root", HadoopDataSource.class.getName()); //$NON-NLS-1$
    conf.set("com.asakusafw.directio.root.path", "/"); //$NON-NLS-1$ //$NON-NLS-2$
    conf.set("com.asakusafw.directio.root.fs.path", getRootPath().toString()); //$NON-NLS-1$
    return conf;
}

From source file:com.asakusafw.operation.tools.directio.file.AbstractFileCopyCommand.java

License:Apache License

@Override
public void run() {
    LOG.debug("starting {}", getClass().getSimpleName());

    if (paths.size() < 2) {
        throw new CommandConfigurationException("source and destination files must be specified");
    }//from   w  ww .j  av a  2 s .  c o  m
    List<DirectIoPath> sources = getSources();
    LOG.debug("source: {}", sources);

    Path destination = getDestination();
    LOG.debug("destination: {}", destination);

    List<ResourceInfo> files = sources.stream().flatMap(it -> {
        List<ResourceInfo> list = FileListCommand.list(it);
        if (list.isEmpty()) {
            throw new CommandConfigurationException(
                    MessageFormat.format("there are no files to copy: {0}", it));
        }
        return list.stream();
    }).collect(Collectors.toList());

    validate(files, destination);
    Optional<FileStatus> stat = stat(destination);

    if (stat.filter(it -> it.isDirectory()).isPresent()) {
        copyOnto(files, destination);
    } else if (stat.filter(it -> it.isDirectory() == false).isPresent()
            && overwriteParameter.isEnabled() == false) {
        throw new CommandConfigurationException(
                MessageFormat.format("destination file already exists: {0}", destination));
    } else {
        Path parent = Optional.ofNullable(destination.getParent())
                .orElseThrow(() -> new IllegalStateException(destination.toString()));
        if (stat(parent).filter(it -> it.isDirectory()).isPresent()) {
            if (sources.size() >= 2) {
                throw new CommandConfigurationException(MessageFormat.format("copy source is ambiguous: {0}",
                        sources.stream().map(String::valueOf).collect(Collectors.joining(", "))));
            }
            copyTo(files.get(0), destination);
        } else {
            throw new CommandConfigurationException(
                    MessageFormat.format("destination directory does not exist: {0}", parent));
        }
    }
}

From source file:com.asakusafw.operation.tools.directio.file.FilePutCommand.java

License:Apache License

@Override
public void run() {
    LOG.debug("starting {}", getClass().getSimpleName());

    if (paths.size() < 2) {
        throw new CommandConfigurationException("source and destination files must be specified");
    }/*from www. java 2 s  .  com*/
    List<java.nio.file.Path> sources = getSources();
    LOG.debug("source: {}", sources);

    org.apache.hadoop.fs.Path destination = getDestination();
    LOG.debug("destination: {}", destination);

    Optional<org.apache.hadoop.fs.FileStatus> stat = stat(destination);
    if (stat.filter(it -> it.isDirectory()).isPresent()) {
        copyOnto(sources, destination);
    } else if (stat.filter(it -> it.isDirectory() == false).isPresent()
            && overwriteParameter.isEnabled() == false) {
        throw new CommandConfigurationException(
                MessageFormat.format("destination file already exists: {0}", destination));
    } else {
        Path parent = Optional.ofNullable(destination.getParent())
                .orElseThrow(() -> new IllegalStateException(destination.toString()));
        if (stat(parent).filter(it -> it.isDirectory()).isPresent()) {
            if (sources.size() >= 2) {
                throw new CommandConfigurationException(MessageFormat.format("copy source is ambiguous: {0}",
                        sources.stream().map(String::valueOf).collect(Collectors.joining(", "))));
            }
            copyTo(sources.get(0), destination);
        } else {
            throw new CommandConfigurationException(
                    MessageFormat.format("destination directory does not exist: {0}", parent));
        }
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopFileFormatAdapter.java

License:Apache License

@Override
public ModelInput<T> createInput(Class<? extends T> dataType, FileSystem fileSystem, final Path path,
        final long offset, final long fragmentSize, Counter counter) throws IOException, InterruptedException {
    FSDataInputStream stream = fileSystem.open(path);
    boolean succeed = false;
    try {//from w  ww  . java  2s  . c  o  m
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format(
                    "Process opening input [stream opened] (path={0}, offset={1}, size={2})", //$NON-NLS-1$
                    path, offset, fragmentSize));
        }
        if (offset != 0) {
            stream.seek(offset);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format(
                        "Process opening input [sought to offset] (path={0}, offset={1}, size={2})", //$NON-NLS-1$
                        path, offset, fragmentSize));
            }
        }
        CountInputStream cstream;
        if (LOG.isDebugEnabled()) {
            cstream = new CountInputStream(stream, counter) {
                @Override
                public void close() throws IOException {
                    LOG.debug(MessageFormat.format("Start closing input (path={0}, offset={1}, size={2})", //$NON-NLS-1$
                            path, offset, fragmentSize));
                    super.close();
                    LOG.debug(MessageFormat.format("Finish closing input (path={0}, offset={1}, size={2})", //$NON-NLS-1$
                            path, offset, fragmentSize));
                }
            };
        } else {
            cstream = new CountInputStream(stream, counter);
        }
        ModelInput<T> input = streamFormat.createInput(dataType, path.toString(), cstream, offset,
                fragmentSize);
        succeed = true;
        return input;
    } finally {
        if (succeed == false) {
            try {
                stream.close();
            } catch (IOException e) {
                LOG.warn(MessageFormat.format("Failed to close input (path={0}, offset={1}, size={2})", path,
                        offset, fragmentSize), e);
            }
        }
    }
}