Example usage for org.apache.hadoop.io MapFile.Writer MapFile.Writer

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapFile.Writer MapFile.Writer.

Prototype

@Deprecated
public Writer(Configuration conf, FileSystem fs, String dirName, WritableComparator comparator,
        Class valClass) throws IOException 

Source Link

Document

Create the named map using the named key comparator.

Usage

From source file:crunch.MaxTemperature.java

License:Apache License

public static void main(String[] args) throws IOException {
        String uri = args[0];/*from   w w w. j av  a 2s .  c  o  m*/
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri), conf);

        IntWritable key = new IntWritable();
        Text value = new Text();
        MapFile.Writer writer = null;
        try {
            writer = new MapFile.Writer(conf, fs, uri, key.getClass(), value.getClass());

            for (int i = 0; i < 1024; i++) {
                key.set(i + 1);
                value.set(DATA[i % DATA.length]);
                writer.append(key, value);
            }
        } finally {
            IOUtils.closeStream(writer);
        }
    }

---

From source file:org.archive.nutchwax.tools.ParseTextCombiner.java

License:Apache License

/**
 *
 *//*from   w  w  w . j  a  v  a2 s  . c  o m*/
public int run(String[] args) throws Exception {
    String usage = "Usage: ParseTextCombiner [-v] output input...\n";

    if (args.length < 1) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    if (args[0].equals("-h")) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    int argStart = 0;
    if (args[argStart].equals("-v")) {
        verbose = true;
        argStart = 1;
    }

    if (args.length - argStart < 2) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);

    Path outputPath = new Path(args[argStart]);
    if (fs.exists(outputPath)) {
        System.err.println("ERROR: output already exists: " + outputPath);
        return -1;
    }

    MapFile.Reader[] readers = new MapFile.Reader[args.length - argStart - 1];
    for (int pos = argStart + 1; pos < args.length; pos++) {
        readers[pos - argStart - 1] = new MapFile.Reader(fs, args[pos], conf);
    }

    WritableComparable[] keys = new WritableComparable[readers.length];
    Writable[] values = new Writable[readers.length];

    WritableComparator wc = WritableComparator.get((Class<WritableComparable>) readers[0].getKeyClass());

    MapFile.Writer writer = new MapFile.Writer(conf, fs, outputPath.toString(),
            (Class<WritableComparable>) readers[0].getKeyClass(), readers[0].getValueClass());

    int readCount = 0;
    int writeCount = 0;

    for (int i = 0; i < readers.length; i++) {
        WritableComparable key = (WritableComparable) ReflectionUtils.newInstance(readers[i].getKeyClass(),
                conf);
        Writable value = (Writable) ReflectionUtils.newInstance(readers[i].getValueClass(), conf);

        if (readers[i].next(key, value)) {
            keys[i] = key;
            values[i] = value;

            readCount++;
            if (verbose)
                System.out.println("read: " + i + ": " + key);
        } else {
            // Not even one key/value pair in the map.
            System.out.println("WARN: No key/value pairs in mapfile: " + args[i + argStart + 1]);
            try {
                readers[i].close();
            } catch (IOException ioe) {
                /* Don't care */ }
            readers[i] = null;
        }
    }

    while (true) {
        int candidate = -1;

        for (int i = 0; i < keys.length; i++) {
            if (keys[i] == null)
                continue;

            if (candidate < 0) {
                candidate = i;
            } else if (wc.compare(keys[i], keys[candidate]) < 0) {
                candidate = i;
            }
        }

        if (candidate < 0) {
            if (verbose)
                System.out.println("Candidate < 0, all done.");
            break;
        }

        // Candidate is the index of the "smallest" key.

        // Write it out.
        writer.append(keys[candidate], values[candidate]);
        writeCount++;
        if (verbose)
            System.out.println("write: " + candidate + ": " + keys[candidate]);

        // Now read in a new value from the corresponding reader.
        if (!readers[candidate].next(keys[candidate], values[candidate])) {
            if (verbose)
                System.out.println(
                        "No more key/value pairs in (" + candidate + "): " + args[candidate + argStart + 1]);

            // No more key/value pairs left in this reader.
            try {
                readers[candidate].close();
            } catch (IOException ioe) {
                /* Don't care */ }
            readers[candidate] = null;
            keys[candidate] = null;
            values[candidate] = null;
        } else {
            readCount++;
            if (verbose)
                System.out.println("read: " + candidate + ": " + keys[candidate]);
        }
    }

    System.out.println("Total # records in : " + readCount);
    System.out.println("Total # records out: " + writeCount);

    writer.close();

    return 0;
}

---

From source file:org.hadoop.tdg.TestPseudoHadoop.java

License:Apache License

/**
 * sorted sequence file// www  .  j av a  2 s.  c o m
 *
 * @throws IOException
 */
@Test
public void mapFileIO() throws IOException {
    LongWritable key = new LongWritable();
    Text value = new Text();
    MapFile.Writer writer = null;
    try {
        writer = new MapFile.Writer(fs.getConf(), fs, DST, key.getClass(), value.getClass());
        for (int i = 0; i < 100; i++) {
            key.set(i);
            value.set(DATA[i % DATA.length]);
            writer.append(key, value);
        }
    } finally {
        IOUtils.closeStream(writer);
    }

    MapFile.Reader reader = null;
    try {
        reader = new MapFile.Reader(fs, DST, fs.getConf());
        LongWritable readerKey = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf());
        Text readerValue = (Text) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf());
        while (reader.next(readerKey, readerValue)) {
            System.out.printf("%s\t%s\n", readerKey, readerValue);
        }
    } finally {
        IOUtils.closeStream(writer);
    }
}

---