Example usage for org.apache.hadoop.fs Path Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(Path parent, Path child)

Source Link

Document

Create a new Path based on the child path resolved against the parent path.

Usage

From source file:DisplayFuzzyKMeans.java

License:Apache License

private static void runSequentialFuzzyKClusterer(Configuration conf, Path samples, Path output,
        DistanceMeasure measure, int maxIterations, float m, double threshold)
        throws IOException, ClassNotFoundException, InterruptedException {
    Path clustersIn = new Path(output, "random-seeds");
    RandomSeedGenerator.buildRandom(conf, samples, clustersIn, 3, measure);
    FuzzyKMeansDriver.run(samples, clustersIn, output, threshold, maxIterations, m, true, true, threshold,
            true);/*from  www.  j  av  a2  s . com*/

    loadClustersWritable(output);
}

From source file:ac.keio.sslab.nlp.lda.RowIdJob.java

License:Apache License

@SuppressWarnings("deprecation")
@Override/*from   ww  w . j  a  v  a  2 s  .c  om*/
public int run(String[] args) throws Exception {

    addInputOption();
    addOutputOption();

    Map<String, List<String>> parsedArgs = parseArguments(args);
    if (parsedArgs == null) {
        return -1;
    }

    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);

    Path outputPath = getOutputPath();
    Path indexPath = new Path(outputPath, "docIndex");
    Path matrixPath = new Path(outputPath, "matrix");

    try (SequenceFile.Writer indexWriter = SequenceFile.createWriter(fs, conf, indexPath, IntWritable.class,
            Text.class);
            SequenceFile.Writer matrixWriter = SequenceFile.createWriter(fs, conf, matrixPath,
                    IntWritable.class, VectorWritable.class)) {
        IntWritable docId = new IntWritable();
        int i = 0;
        int numCols = 0;
        for (Pair<Text, VectorWritable> record : new SequenceFileDirIterable<Text, VectorWritable>(
                getInputPath(), PathType.LIST, PathFilters.logsCRCFilter(), null, true, conf)) {
            VectorWritable value = record.getSecond();
            docId.set(i);
            indexWriter.append(docId, record.getFirst());
            matrixWriter.append(docId, value);
            i++;
            numCols = value.get().size();
        }

        log.info("Wrote out matrix with {} rows and {} columns to {}", i, numCols, matrixPath);
        return 0;
    }
}

From source file:action.hadoop.chapter6.section3.demo632.TestJdbcDriver.java

License:Apache License

public TestJdbcDriver(String name) {
    super(name);/*from  w ww.  ja  v  a  2  s .  c  o  m*/
    conf = new HiveConf(TestJdbcDriver.class);
    String dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", "");
    dataFilePath = new Path(dataFileDir, "kv1.txt");
    dataTypeDataFilePath = new Path(dataFileDir, "datatypes.txt");
    standAloneServer = "true".equals(System.getProperty("test.service.standalone.server"));
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getControlDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_control");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getWriteDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_write");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getReadDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_read");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getAppendDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_append");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getRandomReadDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_random_read");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

private static Path getDataDir(org.apache.hadoop.conf.Configuration conf) {
    return new Path(getBaseDir(conf), "io_data");
}

From source file:alluxio.client.hadoop.DFSIOIntegrationTest.java

License:Apache License

@SuppressWarnings("deprecation")
private void createControlFile(org.apache.hadoop.fs.FileSystem fs, long nrBytes, // in bytes
        int nrFiles) throws IOException {
    LOG.info("creating control file: " + nrBytes + " bytes, " + nrFiles + " files");

    Path controlDir = getControlDir(mConfig);

    if (!fs.exists(controlDir)) {

        fs.delete(controlDir, true);/* ww  w .j a  v  a  2 s  .com*/

        for (int i = 0; i < nrFiles; i++) {
            String name = getFileName(i);
            Path controlFile = new Path(controlDir, "in_file_" + name);
            SequenceFile.Writer writer = null;
            try {
                writer = SequenceFile.createWriter(fs, mConfig, controlFile, Text.class, LongWritable.class,
                        CompressionType.NONE);
                writer.append(new Text(name), new LongWritable(nrBytes));
            } catch (Exception e) {
                throw new IOException(e.getLocalizedMessage());
            } finally {
                if (writer != null) {
                    writer.close();
                }
                writer = null;
            }
        }
    }
    LOG.info("created control files for: " + nrFiles + " files");
}