Example usage for org.apache.hadoop.io MapWritable keySet

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable keySet.

Prototype

@Override
    public Set<Writable> keySet() 

Source Link

Usage

From source file:full_MapReduce.FindBestAttributeMapper.java

License:Open Source License

private double global_entropy(MapWritable data, int tot_tuple) {
    double res = 0.0;

    Map<Text, Integer> count_per_class = new HashMap<Text, Integer>();
    for (Writable tmp_cur_map : data.values()) {
        MapWritable cur_map = (MapWritable) tmp_cur_map;
        for (Writable cur_key : cur_map.keySet()) {
            Text cur_key_text = (Text) cur_key;
            if (!count_per_class.containsKey(cur_key_text)) {
                count_per_class.put(new Text(cur_key_text), 0);
            }/*w  w  w .  j  ava2  s. com*/

            count_per_class.put(cur_key_text,
                    ((IntWritable) cur_map.get(cur_key)).get() + count_per_class.get(cur_key_text));
        }
    }

    double p;
    for (Integer i : count_per_class.values()) {
        p = (i * 1.0) / tot_tuple;
        res -= p * Math.log(p) / Math.log(2);
    }

    return res;
}

---

From source file:full_MapReduce.FindBestAttributeMapper.java

License:Open Source License

private double gain(double global_entropy, Map<Text, Integer> tuple_per_split, MapWritable data,
        int tot_tuple) {
    double sum_partial_entropy = 0;

    Text my_text_key;/*w ww .  j a va  2s  .  c o m*/
    int nb_tuple;
    double uniform_ratio;
    double p;
    for (Writable my_key : data.keySet()) {
        my_text_key = (Text) my_key;
        nb_tuple = tuple_per_split.get(my_text_key);
        uniform_ratio = (nb_tuple * 1.0) / tot_tuple;

        for (Writable my_count : ((MapWritable) data.get(my_key)).values()) {
            p = (((IntWritable) my_count).get() * 1.0) / nb_tuple;
            sum_partial_entropy -= uniform_ratio * p * Math.log(p) / Math.log(2);
        }

    }

    return global_entropy - sum_partial_entropy;
}

---

From source file:gaffer.accumulostore.key.core.AbstractCoreKeyAccumuloElementConverter.java

License:Apache License

@Override
public Properties getPropertiesFromValue(final String group, final Value value)
        throws AccumuloElementConversionException {
    final Properties properties = new Properties();
    if (value == null || value.getSize() == 0) {
        return properties;
    }//from www  .  ja v  a  2  s  .  co m
    final MapWritable map = new MapWritable();
    try (final InputStream inStream = new ByteArrayInputStream(value.get());
            final DataInputStream dataStream = new DataInputStream(inStream)) {
        map.readFields(dataStream);
    } catch (final IOException e) {
        throw new AccumuloElementConversionException("Failed to read map writable from value", e);
    }
    final StoreElementDefinition elementDefinition = storeSchema.getElement(group);
    if (null == elementDefinition) {
        throw new AccumuloElementConversionException("No StoreElementDefinition found for group " + group
                + " is this group in your Store Schema or do your table iterators need updating?");
    }
    for (final Writable writeableKey : map.keySet()) {
        final String propertyName = writeableKey.toString();
        final BytesWritable propertyValueBytes = (BytesWritable) map.get(writeableKey);
        try {
            properties.put(propertyName, elementDefinition.getProperty(propertyName).getSerialiser()
                    .deserialise(propertyValueBytes.getBytes()));
        } catch (final SerialisationException e) {
            throw new AccumuloElementConversionException("Failed to deserialise property " + propertyName, e);
        }
    }
    return properties;
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetails() throws Exception {
    deployTestIndices(1, 1);//  www  . j  av a  2 s  .  c om
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit);
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNotNull(details.get(new Text("category")));
    }
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsWithFieldNames() throws Exception {
    deployTestIndices(1, 1);//from   w  w w .j ava2s  .com
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit, new String[] { "path" });
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNull(details.get(new Text("category")));
    }
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetBinaryDetails() throws Exception {
    File index = _temporaryFolder.newFolder("indexWithBinaryData");
    String textFieldName = "textField";
    String binaryFieldName = "binaryField";
    String textFieldContent = "sample text";
    byte[] bytesFieldContent = new byte[] { 1, 2, 3 };

    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(index), new StandardAnalyzer(Version.LUCENE_35),
            true, MaxFieldLength.UNLIMITED);
    Document document = new Document();
    document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES));
    document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED));
    indexWriter.addDocument(document);//from  w  w  w  .  j  av a 2 s.  c  o m
    indexWriter.optimize();
    indexWriter.close();
    DeployClient deployClient = new DeployClient(_miniCluster.getProtocol());
    IndexState indexState = deployClient.addIndex(index.getName(), index.getParentFile().getAbsolutePath(), 1)
            .joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer())
            .parse(textFieldName + ": " + textFieldContent);
    final Hits hits = client.search(query, new String[] { index.getName() }, 10);
    assertNotNull(hits);
    assertEquals(1, hits.getHits().size());
    final Hit hit = hits.getHits().get(0);
    final MapWritable details = client.getDetails(hit);
    final Set<Writable> keySet = details.keySet();
    assertEquals(1, keySet.size());
    final Writable writable = details.get(new Text(binaryFieldName));
    assertNotNull(writable);
    assertThat(writable, instanceOf(BytesWritable.class));
    BytesWritable bytesWritable = (BytesWritable) writable;
    bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns
    // the full array
    assertArrayEquals(bytesFieldContent, bytesWritable.getBytes());
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsConcurrently() throws KattaException, ParseException, InterruptedException {
    deployTestIndices(1, 1);/*from  w w w .j  a v a2s .  co  m*/
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    List<MapWritable> detailList = client.getDetails(hits.getHits());
    assertEquals(hits.getHits().size(), detailList.size());
    for (int i = 0; i < detailList.size(); i++) {
        final MapWritable details1 = client.getDetails(hits.getHits().get(i));
        final MapWritable details2 = detailList.get(i);
        assertEquals(details1.entrySet(), details2.entrySet());
        final Set<Writable> keySet = details2.keySet();
        assertFalse(keySet.isEmpty());
        final Writable writable = details2.get(new Text("path"));
        assertNotNull(writable);
    }
    client.close();
}

---

From source file:org.apache.nutch.crawl.CrawlDatum.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    byte version = in.readByte(); // read version
    if (version > CUR_VERSION) // check version
        throw new VersionMismatchException(CUR_VERSION, version);

    status = in.readByte();/*from w  w w.  j  a  v a 2s  .  c  om*/
    fetchTime = in.readLong();
    retries = in.readByte();
    if (version > 5) {
        fetchInterval = in.readInt();
    } else
        fetchInterval = Math.round(in.readFloat());
    score = in.readFloat();
    if (version > 2) {
        modifiedTime = in.readLong();
        int cnt = in.readByte();
        if (cnt > 0) {
            signature = new byte[cnt];
            in.readFully(signature);
        } else
            signature = null;
    }

    if (version > 3) {
        boolean hasMetadata = false;
        if (version < 7) {
            org.apache.hadoop.io.MapWritable oldMetaData = new org.apache.hadoop.io.MapWritable();
            if (in.readBoolean()) {
                hasMetadata = true;
                metaData = new org.apache.hadoop.io.MapWritable();
                oldMetaData.readFields(in);
            }
            for (Writable key : oldMetaData.keySet()) {
                metaData.put(key, oldMetaData.get(key));
            }
        } else {
            if (in.readBoolean()) {
                hasMetadata = true;
                metaData = new org.apache.hadoop.io.MapWritable();
                metaData.readFields(in);
            }
        }
        if (hasMetadata == false)
            metaData = null;
    }
    // translate status codes
    if (version < 5) {
        if (oldToNew.containsKey(status))
            status = oldToNew.get(status);
        else
            status = STATUS_DB_UNFETCHED;

    }
}

---

From source file:org.apache.pirk.utils.StringUtils.java

License:Apache License

/**
 * Method to convert a MapWritable into a JSON string
 * //from  www.  j  a va 2s .  c o m
 */
@SuppressWarnings("unchecked")
public static String mapWritableToString(MapWritable map) {
    // Convert to JSON and then write to a String - ensures JSON read-in compatibility
    JSONObject jsonObj = new JSONObject();
    for (Writable key : map.keySet()) {
        jsonObj.put(key.toString(), map.get(key).toString());
    }

    return jsonObj.toJSONString();
}

---

From source file:org.bigsolr.hadoop.SolrRecordWriter.java

License:Apache License

@Override
public void write(NullWritable key, Writable value) throws IOException {

    log.info("SolrRecordWriter ->  write");

    if (solr == null) {
        solr = SolrOperations.getSolrServer(conf);
    }/*from   w  ww.j a  v  a  2 s  . co  m*/

    SolrInputDocument doc = new SolrInputDocument();
    if (value.getClass().getName().equals("org.apache.hadoop.io.MapWritable")) {
        MapWritable valueMap = (MapWritable) value;

        for (Writable keyWritable : valueMap.keySet()) {
            String fieldName = keyWritable.toString();
            Object fieldValue = valueMap.get(new Text(fieldName));
            // Need to add proper conversion of object to Schema field type
            doc.addField(fieldName, fieldValue.toString());
        }
    } else if (value.getClass().getName().equals("org.bigsolr.hadoop.SolrInputRecord")) {
        doc = (SolrInputDocument) value;
    } else {
        log.error("SolrRecordWriter write() Class for Value is not Supported: " + value.getClass().getName());
        System.exit(0);
    }

    try {
        solr.add(doc);
        //solr.commit(true,true);
    } catch (SolrServerException e) {
        log.error("SolrRecordWriter-- solr.add(doc) failed");
        throw new IOException(e);
    }

}

---

• «
• 1
• 2
• 3
• »