Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:Script.java

License:Open Source License

/** Serialize the Javascript object into a file on HDFS and then add
 *  the file to the distributed cache.//from  w  w w. j a v  a  2  s .co  m
 *  @param conf       The Hadoop configuration object
 *  @param o          The Javascript object to serialize
 *  @param name       The name of file to save the serialized object to
 */
public void serialize(Configuration conf, Object o, String name) throws IOException {
    FileSystem hdfs = FileSystem.get(conf);
    Path path = new Path(Eggshell.SCRIPT_DIR + "/" + name);
    FSDataOutputStream out = hdfs.create(path); // create the file
    String buf;
    if (!(o instanceof NativeObject)) {
        buf = cx.toString(o); // serialize
        if (o instanceof NativeArray)
            buf = "[" + buf + "]"; // if array
    } else {
        buf = "{";
        NativeObject obj = (NativeObject) o;
        Object[] propIds = obj.getPropertyIds(obj);
        for (Object propId : propIds) {
            String key = propId.toString();
            Object value = obj.getProperty(obj, key);
            buf += key + ":" + cx.toString(value) + ",";
        }
        buf += "}";
    }
    buf = "(" + buf + ")"; // force evaluation
    out.writeUTF(buf);
    out.close();
    DistributedCache.addCacheFile(path.toUri(), conf);
}

From source file:Vectors.java

License:Apache License

public static void write(Vector vector, Path path, Configuration conf, boolean laxPrecision)
        throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    FSDataOutputStream out = fs.create(path);
    try {/*from w  w w  .j a va 2 s.co m*/
        VectorWritable vectorWritable = new VectorWritable(vector);
        vectorWritable.setWritesLaxPrecision(laxPrecision);
        vectorWritable.write(out);
    } finally {
        Closeables.closeQuietly(out);
    }
}

From source file:Vectors.java

License:Apache License

public static OpenIntIntHashMap readAsIntMap(Path path, Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    FSDataInputStream in = fs.open(path);
    try {//from  ww  w .  ja v a  2s  .c om
        return readAsIntMap(in);
    } finally {
        Closeables.closeQuietly(in);
    }
}

From source file:Vectors.java

License:Apache License

public static Vector read(Path path, Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    FSDataInputStream in = fs.open(path);
    try {/*from  ww  w . j  a  v  a2 s .  com*/
        return VectorWritable.readVector(in);
    } finally {
        Closeables.closeQuietly(in);
    }
}

From source file:DisplayClustering.java

License:Apache License

protected static void writeSampleData(Path output) throws IOException {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(output.toUri(), conf);
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, Text.class, VectorWritable.class);
    try {//from   ww  w . j  av a  2  s. c  o  m
        int i = 0;
        for (VectorWritable vw : SAMPLE_DATA) {
            writer.append(new Text("sample_" + i++), vw);
        }
    } finally {
        Closeables.close(writer, false);
    }
}

From source file:DisplayClustering.java

License:Apache License

protected static void loadClustersWritable(Path output) throws IOException {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(output.toUri(), conf);
    for (FileStatus s : fs.listStatus(output, new ClustersFilter())) {
        List<Cluster> clusters = readClustersWritable(s.getPath());
        CLUSTERS.add(clusters);//from   w  w  w .  j a  va  2s. co m
    }
}

From source file:RawParascaleFileSystem.java

License:Apache License

/**
 * {@inheritDoc}//from  w  ww  .j a v a 2s  .com
 */
@Override
public File pathToFile(final Path path) {
    // this has public modifier due to subclassing
    checkPath(path);
    final String mountPoint = getMountPoint();
    final File file;

    if (path.isAbsolute()) {
        final URI pathUri = path.makeQualified(this.getUri(), this.getWorkingDirectory()).toUri();
        final String[] parseAuthority = parseAuthority(pathUri);

        file = new File(mountPoint,
                String.format("/%s/%s/%s", parseAuthority[1], parseAuthority[0], pathUri.getPath()));

    } else {
        file = new File(mountPoint, String.format("/%s/%s/%s/%s", controlNode, virtualFs, getRawHomeDirectory(),
                path.toUri().getPath()));
    }
    return file;
}

From source file:alluxio.hadoop.HadoopUtils.java

License:Apache License

/**
 * Given a {@link Path} path, it returns the path component of its URI, which has the form
 * scheme://authority/path.//from  w  w  w  .  j  ava 2 s .  c  o m
 *
 * @param path an HDFS {@link Path}
 * @return the path component of the {@link Path} URI
 */
public static String getPathWithoutScheme(Path path) {
    return path.toUri().getPath();
}

From source file:alluxio.yarn.YarnUtils.java

License:Apache License

/**
 * Creates a local resource for a file on HDFS.
 *
 * @param yarnConf YARN configuration//  w  w  w  . java2s  .  c o m
 * @param resource the path to a resource file on HDFS
 * @throws IOException if the file can not be found on HDFS
 * @return the created local resource
 */
public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resource)
        throws IOException {
    LocalResource localResource = Records.newRecord(LocalResource.class);

    Path resourcePath = new Path(resource);

    FileStatus jarStat = FileSystem.get(resourcePath.toUri(), yarnConf).getFileStatus(resourcePath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
    return localResource;
}

From source file:apex.benchmark.RedisHelper.java

License:Apache License

public void fillDB(String fileName) throws IOException {
    Path filePath = new Path(fileName);
    Configuration configuration = new Configuration();
    FileSystem fs;/*from  w  w w  .  j a  v  a2  s.c om*/
    fs = FileSystem.newInstance(filePath.toUri(), configuration);
    FSDataInputStream inputStream = fs.open(filePath);
    BufferedReader bufferedReader;

    try {

        bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

        String line;
        while ((line = bufferedReader.readLine()) != null) {

            String[] mapping = line.split("\\s+");

            if (mapping.length != 2) {
                continue;
            }

            jedis.sadd("campaigns", mapping[0]);
            jedis.set(mapping[1], mapping[0]);
        }
    } catch (Exception e) {
        throw e;
    }
}