Example usage for org.apache.hadoop.io SequenceFile.Writer SequenceFile.Writer

Introduction

In this page you can find the example usage for org.apache.hadoop.io SequenceFile.Writer SequenceFile.Writer.

Prototype

@Deprecated
public Writer(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass)
        throws IOException 

Source Link

Document

Create the named file.

Usage

From source file:org.apache.mahout.classifier.sequencelearning.baumwelchmapreduce.MapWritableCache.java

License:Apache License

/**
 * @param key SequenceFile key//from ww  w. j a v a2  s .  c  o m
 * @param map Map to save
 */
public static void save(Writable key, MapWritable map, Path output, Configuration conf, boolean overwritePath,
        boolean deleteOnExit) throws IOException {

    FileSystem fs = FileSystem.get(conf);
    output = fs.makeQualified(output);
    if (overwritePath) {
        HadoopUtil.delete(conf, output);
    }

    // set the cache
    DistributedCache.setCacheFiles(new URI[] { output.toUri() }, conf);

    // set up the writer
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, IntWritable.class,
            MapWritable.class);
    try {
        writer.append(key, new MapWritable(map));
    } finally {
        Closeables.closeQuietly(writer);
    }

    if (deleteOnExit) {
        fs.deleteOnExit(output);
    }
}

---

From source file:org.apache.mrql.Evaluator.java

License:Apache License

/** dump MRQL data into a sequence file */
public void dump(String file, Tree type, MRData data) throws Exception {
    Path path = new Path(file);
    FileSystem fs = path.getFileSystem(Plan.conf);
    PrintStream ftp = new PrintStream(fs.create(path.suffix(".type")));
    ftp.print("2@" + type.toString() + "\n");
    ftp.close();//from w  ww.j a  va 2 s .com
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, Plan.conf, path, MRContainer.class,
            MRContainer.class);
    if (data instanceof MR_dataset)
        data = Plan.collect(((MR_dataset) data).dataset());
    if (data instanceof Bag) {
        Bag s = (Bag) data;
        long i = 0;
        for (MRData e : s) {
            counter_key.set(i++);
            value_container.set(e);
            writer.append(counter_container, value_container);
        }
    } else {
        counter_key.set(0);
        value_container.set(data);
        writer.append(counter_container, value_container);
    }
    ;
    writer.close();
}

---

From source file:org.apache.mrql.Plan.java

License:Apache License

/** The cache operator that dumps a bag into an HDFS file */
public final static DataSet fileCache(Bag s) throws IOException {
    String newpath = new_path(conf);
    Path path = new Path(newpath);
    FileSystem fs = path.getFileSystem(conf);
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, MRContainer.class, MRContainer.class);
    long i = 0;/*from  w w w. j av  a  2  s  .c o  m*/
    for (MRData e : s) {
        counter_key.set(i++);
        value_container.set(e);
        writer.append(counter_container, value_container);
    }
    ;
    writer.close();
    return new DataSet(new BinaryDataSource(0, newpath, conf), 0, 0);
}

---