Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString()

Source Link

Usage

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

private static void openCommentMap() {
    final Configuration conf = new Configuration();
    try {/*from   w  w  w .  ja  v  a2s. c om*/
        final FileSystem fs = FileSystem.get(conf);
        final Path p = new Path("hdfs://boa-njt/",
                new Path(
                        context.getConfiguration().get("boa.comments.dir",
                                context.getConfiguration().get("boa.input.dir", "repcache/live")),
                        new Path("comments")));
        commentsMap = new MapFile.Reader(fs, p.toString(), conf);
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

private static void openIssuesMap() {
    final Configuration conf = new Configuration();
    try {/*from  w w  w  .  j a  v  a  2  s. co  m*/
        final FileSystem fs = FileSystem.get(conf);
        final Path p = new Path("hdfs://boa-njt/",
                new Path(
                        context.getConfiguration().get("boa.issues.dir",
                                context.getConfiguration().get("boa.input.dir", "repcache/live")),
                        new Path("issues")));
        issuesMap = new MapFile.Reader(fs, p.toString(), conf);
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

From source file:ca.uwaterloo.iss4e.hadoop.io.CartesianInputFormat.java

License:Open Source License

private List<InputSplit> getInputSplits(JobContext jobContext, String inputFormatClass, Path path)
        throws ClassNotFoundException, IOException {
    Configuration conf = jobContext.getConfiguration();
    FileInputFormat inputFormat = (FileInputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatClass),
            conf);/*from w ww  . j ava2 s .c  o  m*/

    // Set the input path for the left data set
    path = path.getFileSystem(conf).makeQualified(path);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get(INPUT_DIR);
    conf.set(INPUT_DIR, dirStr);
    return inputFormat.getSplits(jobContext);
}

From source file:ca.uwaterloo.iss4e.spark.pointperrow.CosineMain.java

License:Open Source License

public void readFiles(JavaSparkContext sc, FileSystem fs, FileStatus[] files) {
    for (int i = 0; i < files.length; i++) {
        if (files[i].isDirectory()) {
            try {
                readFiles(sc, fs, fs.listStatus(files[i].getPath()));
            } catch (IOException e) {
                e.printStackTrace();// w ww.  jav  a 2s .  co  m
            }
        } else {
            if (lines == null) {
                Path p = files[i].getPath();
                lines = sc.textFile(p.toString());
            } else {
                JavaRDD<String> r = sc.textFile(files[i].getPath().toString());
                lines.union(r);
            }
        }
    }
}

From source file:cascading.flow.hadoop.MapReduceFlow.java

License:Open Source License

protected Map<String, Tap> createSources(JobConf jobConf) {
    Path[] paths = FileInputFormat.getInputPaths(jobConf);

    if (paths.length == 0) {
        try {//from ww  w  .  ja v  a  2s  . c om
            paths = org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getInputPaths(new Job(jobConf));
        } catch (IOException exception) {
            throw new CascadingException(exception);
        }
    }

    Map<String, Tap> taps = new HashMap<String, Tap>();

    for (Path path : paths)
        taps.put(path.toString(), new Hfs(new NullScheme(), path.toString()));

    return taps;
}

From source file:cascading.flow.hadoop.MapReduceFlow.java

License:Open Source License

protected Map<String, Tap> createSinks(JobConf jobConf) {
    Map<String, Tap> taps = new HashMap<String, Tap>();

    Path path = FileOutputFormat.getOutputPath(jobConf);

    if (path == null) {
        try {//w  ww.j a va2  s. c  om
            path = org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.getOutputPath(new Job(jobConf));
        } catch (IOException exception) {
            throw new CascadingException(exception);
        }
    }

    taps.put(path.toString(),
            new Hfs(new NullScheme(), path.toString(), deleteSinkOnInit ? SinkMode.REPLACE : SinkMode.KEEP));

    return taps;
}

From source file:cascading.flow.hadoop.util.HadoopMRUtil.java

License:Open Source License

public static String readStateFromDistCache(JobConf jobConf, String id, String kind) throws IOException {
    Path[] files = DistributedCache.getLocalCacheFiles(jobConf);

    Path stepStatePath = null;/*from w ww  .j a v a  2 s  .co  m*/

    for (Path file : files) {
        if (!file.toString().contains(kind + "-state-" + id))
            continue;

        stepStatePath = file;
        break;
    }

    if (stepStatePath == null)
        throw new FlowException("unable to find step state from distributed cache");

    LOG.info("reading step state from local path: {}", stepStatePath);

    Hfs temp = new Lfs(new TextLine(new Fields("line")), stepStatePath.toString());

    TupleEntryIterator reader = null;

    try {
        reader = temp.openForRead(new HadoopFlowProcess(jobConf));

        if (!reader.hasNext())
            throw new FlowException("step state path is empty: " + temp.getIdentifier());

        return reader.next().getString(0);
    } catch (IOException exception) {
        throw new FlowException("unable to find state path: " + temp.getIdentifier(), exception);
    } finally {
        try {
            if (reader != null)
                reader.close();
        } catch (IOException exception) {
            LOG.warn("error closing state path reader", exception);
        }
    }
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void addInputPath(Configuration conf, Path path) {
    Path workingDirectory = getWorkingDirectory(conf);
    path = new Path(workingDirectory, path);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get("mapred.input.dir");
    conf.set("mapred.input.dir", dirs == null ? dirStr : dirs + StringUtils.COMMA_STR + dirStr);
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void setOutputPath(Configuration conf, Path path) {
    Path workingDirectory = getWorkingDirectory(conf);
    path = new Path(workingDirectory, path);
    conf.set("mapred.output.dir", path.toString());
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

private static Path getWorkingDirectory(Configuration conf) {
    String name = conf.get("mapred.working.dir");
    if (name != null) {
        return new Path(name);
    } else {/*from  w  w w .  j a v a  2  s. c om*/
        try {
            Path dir = FileSystem.get(conf).getWorkingDirectory();
            conf.set("mapred.working.dir", dir.toString());
            return dir;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}