Example usage for org.apache.hadoop.io WritableComparator compare

Introduction

In this page you can find the example usage for org.apache.hadoop.io WritableComparator compare.

Prototype

@Override
    public int compare(Object a, Object b) 

Source Link

Usage

From source file:com.asakusafw.runtime.io.util.WritableTestRoot.java

License:Apache License

static int cmp(WritableRawComparable a, WritableRawComparable b, WritableComparator comp) throws IOException {
    int cmp = comp.compare(a, b);
    byte[] serA = ser(a);
    byte[] serB = ser(b);
    int serCmp = comp.compare(serA, 0, serA.length, serB, 0, serB.length);
    assertThat(serCmp, is(cmp));/*from www.  j  ava  2s. c o  m*/
    return cmp;
}

---

From source file:org.apache.hama.bsp.join.OverrideRecordReader.java

License:Apache License

/**
 * Instead of filling the JoinCollector with iterators from all data sources,
 * fill only the rightmost for this key. This not only saves space by
 * discarding the other sources, but it also emits the number of key-value
 * pairs in the preferred RecordReader instead of repeating that stream n
 * times, where n is the cardinality of the cross product of the discarded
 * streams for the given key./*from  w  w w. j ava  2s.  c o  m*/
 */
protected void fillJoinCollector(K iterkey) throws IOException {
    final PriorityQueue<ComposableRecordReader<K, ?>> q = getRecordReaderQueue();
    if (!q.isEmpty()) {
        int highpos = -1;
        ArrayList<ComposableRecordReader<K, ?>> list = new ArrayList<ComposableRecordReader<K, ?>>(kids.length);
        q.peek().key(iterkey);
        final WritableComparator cmp = getComparator();
        while (0 == cmp.compare(q.peek().key(), iterkey)) {
            ComposableRecordReader<K, ?> t = q.poll();
            if (-1 == highpos || list.get(highpos).id() < t.id()) {
                highpos = list.size();
            }
            list.add(t);
            if (q.isEmpty())
                break;
        }
        ComposableRecordReader<K, ?> t = list.remove(highpos);
        t.accept(jc, iterkey);
        for (ComposableRecordReader<K, ?> rr : list) {
            rr.skip(iterkey);
        }
        list.add(t);
        for (ComposableRecordReader<K, ?> rr : list) {
            if (rr.hasNext()) {
                q.add(rr);
            }
        }
    }
}

---

From source file:org.archive.nutchwax.tools.ParseTextCombiner.java

License:Apache License

/**
 *
 *//*from ww  w. j  av  a2s.c o m*/
public int run(String[] args) throws Exception {
    String usage = "Usage: ParseTextCombiner [-v] output input...\n";

    if (args.length < 1) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    if (args[0].equals("-h")) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    int argStart = 0;
    if (args[argStart].equals("-v")) {
        verbose = true;
        argStart = 1;
    }

    if (args.length - argStart < 2) {
        System.err.println("Usage: " + usage);
        return 1;
    }

    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);

    Path outputPath = new Path(args[argStart]);
    if (fs.exists(outputPath)) {
        System.err.println("ERROR: output already exists: " + outputPath);
        return -1;
    }

    MapFile.Reader[] readers = new MapFile.Reader[args.length - argStart - 1];
    for (int pos = argStart + 1; pos < args.length; pos++) {
        readers[pos - argStart - 1] = new MapFile.Reader(fs, args[pos], conf);
    }

    WritableComparable[] keys = new WritableComparable[readers.length];
    Writable[] values = new Writable[readers.length];

    WritableComparator wc = WritableComparator.get((Class<WritableComparable>) readers[0].getKeyClass());

    MapFile.Writer writer = new MapFile.Writer(conf, fs, outputPath.toString(),
            (Class<WritableComparable>) readers[0].getKeyClass(), readers[0].getValueClass());

    int readCount = 0;
    int writeCount = 0;

    for (int i = 0; i < readers.length; i++) {
        WritableComparable key = (WritableComparable) ReflectionUtils.newInstance(readers[i].getKeyClass(),
                conf);
        Writable value = (Writable) ReflectionUtils.newInstance(readers[i].getValueClass(), conf);

        if (readers[i].next(key, value)) {
            keys[i] = key;
            values[i] = value;

            readCount++;
            if (verbose)
                System.out.println("read: " + i + ": " + key);
        } else {
            // Not even one key/value pair in the map.
            System.out.println("WARN: No key/value pairs in mapfile: " + args[i + argStart + 1]);
            try {
                readers[i].close();
            } catch (IOException ioe) {
                /* Don't care */ }
            readers[i] = null;
        }
    }

    while (true) {
        int candidate = -1;

        for (int i = 0; i < keys.length; i++) {
            if (keys[i] == null)
                continue;

            if (candidate < 0) {
                candidate = i;
            } else if (wc.compare(keys[i], keys[candidate]) < 0) {
                candidate = i;
            }
        }

        if (candidate < 0) {
            if (verbose)
                System.out.println("Candidate < 0, all done.");
            break;
        }

        // Candidate is the index of the "smallest" key.

        // Write it out.
        writer.append(keys[candidate], values[candidate]);
        writeCount++;
        if (verbose)
            System.out.println("write: " + candidate + ": " + keys[candidate]);

        // Now read in a new value from the corresponding reader.
        if (!readers[candidate].next(keys[candidate], values[candidate])) {
            if (verbose)
                System.out.println(
                        "No more key/value pairs in (" + candidate + "): " + args[candidate + argStart + 1]);

            // No more key/value pairs left in this reader.
            try {
                readers[candidate].close();
            } catch (IOException ioe) {
                /* Don't care */ }
            readers[candidate] = null;
            keys[candidate] = null;
            values[candidate] = null;
        } else {
            readCount++;
            if (verbose)
                System.out.println("read: " + candidate + ": " + keys[candidate]);
        }
    }

    System.out.println("Total # records in : " + readCount);
    System.out.println("Total # records out: " + writeCount);

    writer.close();

    return 0;
}

---