Example usage for org.apache.hadoop.mapreduce Partitioner getPartition

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Partitioner getPartition.

Prototype

public abstract int getPartition(KEY key, VALUE value, int numPartitions);

Source Link

Document

Get the partition number for a given key (hence record) given the total number of partitions i.e.

Usage

From source file:org.apache.blur.utils.TableShardCountCollapserTest.java

License:Apache License

private void assertData(int totalShardCount) throws IOException {
    Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
    for (int i = 0; i < totalShardCount; i++) {
        HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, ShardUtil.getShardName(i)));
        DirectoryReader reader = DirectoryReader.open(directory);
        int numDocs = reader.numDocs();
        for (int d = 0; d < numDocs; d++) {
            Document document = reader.document(d);
            IndexableField field = document.getField("id");
            Integer id = (Integer) field.numericValue();
            int partition = partitioner.getPartition(new IntWritable(id), null, totalShardCount);
            assertEquals(i, partition);//  ww  w . j  a v  a2  s  .  c om
        }
        reader.close();
    }
}

---

From source file:org.apache.blur.utils.TableShardCountCollapserTest.java

License:Apache License

private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
        throws IOException {
    HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);

    Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
    int partition = partitioner.getPartition(new IntWritable(i), null, totalShardCount);
    assertEquals(i, partition);/*from w w w  .  j  a  va 2  s.co m*/

    Document doc = getDoc(i);
    indexWriter.addDocument(doc);
    indexWriter.close();
}

---

From source file:org.apache.mahout.utils.nlp.collocations.llr.GramKeyPartitionerTest.java

License:Apache License

@Test
public void testPartition() {
    byte[] foo = new byte[1];
    foo[0] = 1;//from   w  w w  .  j  ava 2  s  .c  o  m

    foo[0] = 2;

    byte[] empty = new byte[0];
    GramKey a = new GramKey(new Gram("foo", 1, Gram.Type.HEAD), empty);
    GramKey b = new GramKey(new Gram("foo", 1, Gram.Type.HEAD), foo);
    byte[] bar = new byte[1];
    GramKey c = new GramKey(new Gram("foo", 2, Gram.Type.HEAD), bar);
    GramKey d = new GramKey(new Gram("foo", 1, Gram.Type.TAIL), empty);
    GramKey e = new GramKey(new Gram("foo", 2, Gram.Type.TAIL), foo);

    Partitioner<GramKey, Gram> p = new GramKeyPartitioner();
    int numPartitions = 5;

    int ap = p.getPartition(a, null, numPartitions);
    int bp = p.getPartition(b, null, numPartitions);
    int cp = p.getPartition(c, null, numPartitions);
    int dp = p.getPartition(d, null, numPartitions);
    int ep = p.getPartition(e, null, numPartitions);

    Assert.assertEquals(ap, bp);
    Assert.assertEquals(ap, cp);
    Assert.assertEquals(dp, ep);
}

---

From source file:org.apache.mahout.vectorizer.collocations.llr.GramKeyPartitionerTest.java

License:Apache License

@Test
public void testPartition() {
    byte[] foo = new byte[1];
    foo[0] = 1;/*  w w  w.  j a  v a 2  s . c o  m*/

    foo[0] = 2;

    byte[] empty = new byte[0];
    GramKey a = new GramKey(new Gram("foo", 1, Gram.Type.HEAD), empty);
    GramKey b = new GramKey(new Gram("foo", 1, Gram.Type.HEAD), foo);
    byte[] bar = new byte[1];
    GramKey c = new GramKey(new Gram("foo", 2, Gram.Type.HEAD), bar);
    GramKey d = new GramKey(new Gram("foo", 1, Gram.Type.TAIL), empty);
    GramKey e = new GramKey(new Gram("foo", 2, Gram.Type.TAIL), foo);

    Partitioner<GramKey, Gram> p = new GramKeyPartitioner();
    int numPartitions = 5;

    int ap = p.getPartition(a, null, numPartitions);
    int bp = p.getPartition(b, null, numPartitions);
    int cp = p.getPartition(c, null, numPartitions);
    int dp = p.getPartition(d, null, numPartitions);
    int ep = p.getPartition(e, null, numPartitions);

    assertEquals(ap, bp);
    assertEquals(ap, cp);
    assertEquals(dp, ep);
}

---

From source file:org.godhuli.rhipe.RHMapFileOutputFormat.java

License:Apache License

/** Get an entry from output generated by this class. */
public static RHBytesWritable getEntry(MapFile.Reader[] readers,
        Partitioner<RHBytesWritable, RHBytesWritable> partitioner, RHBytesWritable key, RHBytesWritable value)
        throws IOException {
    int part = partitioner.getPartition(key, value, readers.length);
    return (RHBytesWritable) readers[part].get(key, value);
}

---

• «
• 1
• 2
• »