Example usage for org.apache.hadoop.io MapWritable keySet

List of usage examples for org.apache.hadoop.io MapWritable keySet

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. MapWritable
  6. keySet

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable keySet.

Prototype

@Override
    public Set<Writable> keySet()

Source Link

Usage

From source file:com.csiro.hadoop.WritableTest.java

public static void main(String[] args) {
    System.out.println("*** Primitive Writable ***");

    BooleanWritable bool1 = new BooleanWritable(true);
    ByteWritable byte1 = new ByteWritable((byte) 3);
    System.out.printf("Boolean:%s Byte:%d\n", bool1, byte1.get());

    IntWritable int1 = new IntWritable(5);
    IntWritable int2 = new IntWritable(17);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    int1.set(int2.get());
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    Integer int3 = new Integer(23);
    int1.set(int3);
    System.out.printf("I1:%d I2:%d\n", int1.get(), int2.get());

    System.out.println("*** Array Writable ***");

    ArrayWritable a = new ArrayWritable(IntWritable.class);
    a.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] values = (IntWritable[]) a.get();
    for (IntWritable i : values) {
        System.out.println(i);//w  ww  .j  a v a 2  s .co m
    }

    IntArrayWritable ia = new IntArrayWritable();
    ia.set(new IntWritable[] { new IntWritable(1), new IntWritable(3), new IntWritable(5) });

    IntWritable[] ivalues = (IntWritable[]) ia.get();

    ia.set((new LongWritable[] { new LongWritable(10001) }));

    System.out.println("*** Map Writables ***");

    MapWritable m = new MapWritable();
    IntWritable key1 = new IntWritable(5);
    NullWritable value1 = NullWritable.get();

    m.put(key1, value1);
    System.out.println(m.containsKey(key1));
    System.out.println(m.get(key1));
    m.put(new LongWritable(100000000), key1);
    Set<Writable> keys = m.keySet();

    for (Writable k : keys)
        System.out.println(k.getClass());

}

From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetails() throws Exception {
    miniCluster.deployIndex(LuceneTestResources.INDEX1, 1);
    LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol());
    final Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, null, 10);
    assertNotNull(hits);/*  w  w w.  ja va2s . com*/
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit);
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNotNull(details.get(new Text("category")));
    }
    client.close();
}

From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsWithFieldNames() throws Exception {
    miniCluster.deployIndex(LuceneTestResources.INDEX1, 1);
    LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol());
    final Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, null, 10);
    assertNotNull(hits);/*from  w  w  w . j av a 2s.  c om*/
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit, new String[] { "path" });
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNull(details.get(new Text("category")));
    }
    client.close();
}

From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java

License:Apache License

@Test
public void testGetBinaryDetails() throws Exception {
    File index = temporaryFolder.newFolder("indexWithBinaryData");
    File indexShard = new File(index, "binaryShard");
    if (!indexShard.mkdirs()) {
        throw new RuntimeException("Unable to create directory " + indexShard.getAbsolutePath());
    }/*from  ww  w . j  a v  a 2 s  .c o m*/

    String textFieldName = "textField";
    String binaryFieldName = "binaryField";
    String textFieldContent = "sample text";
    byte[] bytesFieldContent = new byte[] { 1, 2, 3 };

    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(indexShard),
            new StandardAnalyzer(Version.LUCENE_30), true, MaxFieldLength.UNLIMITED);
    Document document = new Document();
    document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES));
    document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED));
    indexWriter.addDocument(document);
    indexWriter.close(true);
    DeployClient deployClient = new DeployClient(miniCluster.getProtocol());
    IndexState indexState = deployClient.addIndex(index.getName(), index.getAbsolutePath(), 1).joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol());
    final Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer())
            .parse(textFieldName + ": " + textFieldContent);
    final Hits hits = client.search(query, new String[] { index.getName() }, 10);
    assertNotNull(hits);
    assertEquals(1, hits.getHits().size());
    final Hit hit = hits.getHits().get(0);
    final MapWritable details = client.getDetails(hit);
    final Set<Writable> keySet = details.keySet();
    assertEquals(1, keySet.size());
    final Writable writable = details.get(new Text(binaryFieldName));
    assertNotNull(writable);
    assertThat(writable, instanceOf(BytesWritable.class));
    BytesWritable bytesWritable = (BytesWritable) writable;
    bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns
    // the full array
    assertArrayEquals(bytesFieldContent, bytesWritable.getBytes());
    client.close();
}

From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsConcurrently() throws Exception {
    miniCluster.deployIndex(LuceneTestResources.INDEX1, 1);
    LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol());
    final Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, null, 10);
    assertNotNull(hits);/*from  w  w  w .j  a  va 2 s .c om*/
    assertEquals(10, hits.getHits().size());
    List<MapWritable> detailList = client.getDetails(hits.getHits());
    assertEquals(hits.getHits().size(), detailList.size());
    for (int i = 0; i < detailList.size(); i++) {
        final MapWritable details1 = client.getDetails(hits.getHits().get(i));
        final MapWritable details2 = detailList.get(i);
        assertEquals(details1.entrySet(), details2.entrySet());
        final Set<Writable> keySet = details2.keySet();
        assertFalse(keySet.isEmpty());
        final Writable writable = details2.get(new Text("path"));
        assertNotNull(writable);
    }
    client.close();
}

From source file:com.mapred.JoinReducer.java

protected void reduce(DepDatePair key, Iterable<MapWritable> values, Context context)
        throws IOException, InterruptedException {
    Iterator valuesIterator = values.iterator();
    if (valuesIterator.hasNext()) {
        this.index = 1L;
        MapWritable depMap = (MapWritable) valuesIterator.next();
        this.departmentName = depMap.get(MRUtils.DEPARTMENT_NAME).toString();
        if (this.departmentName == null) {
            this.departmentName = "DEP#NAME#ERROR";
        }//www  .  j  a v  a 2  s  .c  o m
        while (valuesIterator.hasNext()) {
            MapWritable map = (MapWritable) valuesIterator.next();
            Set keySet = map.keySet();
            for (Object singleKey : keySet) {
                this.builder.append(map.get((Text) singleKey));
                this.builder.append(",");
            }
            this.builder.append(this.departmentName);
            this.builder.append(";");
            this.outputValue.set(this.builder.toString());
            context.write(new LongWritable(this.index++), outputValue);
            this.builder.delete(0, this.builder.length());
        }
    }

}

From source file:com.shmsoft.dmass.main.Reduce.java

License:Apache License

private Metadata getAllMetadata(MapWritable map) {
    Metadata metadata = new Metadata();
    Set<Writable> set = map.keySet();
    Iterator<Writable> iter = set.iterator();
    while (iter.hasNext()) {
        String name = iter.next().toString();
        if (!ParameterProcessing.NATIVE.equals(name) && !ParameterProcessing.NATIVE_AS_PDF.equals(name)) { // all metadata but native - which is bytes!
            Text value = (Text) map.get(new Text(name));
            metadata.set(name, value.toString());
        }/*from ww w  . j  av a2s .c om*/
    }
    return metadata;
}

From source file:edu.ub.ahstfg.io.WritableConverter.java

License:Open Source License

/**
 * Converts MapWritable to HashMap<String, LinkedList<Long>>.
 * @param input MapWritable to convert./*from   ww w .j a v a2 s  .  co  m*/
 * @return Converted HashMap.
 */
public static HashMap<String, LinkedList<Short>> mapWritable2HashMapStringLinkedListShort(MapWritable input) {
    HashMap<String, LinkedList<Short>> ret = new HashMap<String, LinkedList<Short>>();
    Text t;
    ArrayWritable aw;
    LinkedList<Short> al;
    for (Writable w : input.keySet()) {
        t = (Text) w;
        aw = (ArrayWritable) input.get(t);
        al = arrayWritable2LinkedListShort(aw);
        ret.put(t.toString(), al);
    }
    return ret;
}

From source file:full_MapReduce.FindBestAttributeMapper.java

License:Open Source License

private TextArrayWritable getValues(MapWritable value) {
    TextArrayWritable res = new TextArrayWritable();
    Text[] tmp_res = new Text[value.keySet().size()];

    int index = 0;
    for (Writable w1 : value.keySet()) {
        MapWritable mw = (MapWritable) value.get(w1);
        int nb_class = mw.size();
        Text prefered_class = new Text();
        IntWritable best_count = new IntWritable(Integer.MIN_VALUE);
        for (Writable w2 : mw.keySet()) {
            if (((IntWritable) mw.get(w2)).compareTo(best_count) > 0) {
                best_count = (IntWritable) mw.get(w2);
                prefered_class.set((Text) w2);
            }//w ww.  j  a  v  a  2  s  .c o  m
        }
        tmp_res[index++] = new Text(((Text) w1).toString() + " " + nb_class + " " + prefered_class.toString());
    }

    res.set(tmp_res);
    return res;
}

From source file:full_MapReduce.FindBestAttributeMapper.java

License:Open Source License

private Map<Text, Integer> getTuplePerSplit(MapWritable data) {
    Map<Text, Integer> res = new HashMap<Text, Integer>();

    Text my_text_key;/*from   w  w w . ja  va2  s  .c  om*/
    int nb_tuple;
    for (Writable my_key : data.keySet()) {
        my_text_key = (Text) my_key;
        nb_tuple = 0;

        for (Writable my_value : ((MapWritable) data.get(my_key)).values()) {
            nb_tuple += ((IntWritable) my_value).get();
        }

        res.put(new Text(my_text_key), nb_tuple);
    }

    return res;
}