Example usage for org.apache.hadoop.io MapWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io MapWritable get.

Prototype

@Override
    public Writable get(Object key) 

Source Link

Usage

From source file:in.dream_lab.goffish.LongMapJSONReader.java

License:Apache License

@SuppressWarnings("unchecked")
Vertex<V, E, LongWritable, LongWritable> createVertex(String JSONString) {
    JSONArray JSONInput = (JSONArray) JSONValue.parse(JSONString);

    LongWritable sourceID = new LongWritable(Long.valueOf(JSONInput.get(0).toString()));
    assert (vertexMap.get(sourceID) == null);

    Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>(sourceID);
    //fix this// www.java 2s. co m
    //assumed value of jsonMap= "key1:type1:value1$ key2:type2:value2$....."
    //type could be Long or String or Double
    String jsonMap = JSONInput.get(1).toString();
    String[] vprop = jsonMap.split(Pattern.quote("$"));
    //key,value property pairs for a vertex
    MapWritable vertexMap = new MapWritable();
    for (int i = 0; i < vprop.length; i++) {
        String[] map = vprop[i].split(Pattern.quote(":"));
        Text key = new Text(map[0]);
        //FIXME:assuming String values for now
        Text value = new Text(map[2]);
        vertexMap.put(key, value);
    }

    V vertexValue = (V) vertexMap;

    vertex.setValue(vertexValue);

    JSONArray edgeList = (JSONArray) JSONInput.get(2);
    for (Object edgeInfo : edgeList) {
        Object edgeValues[] = ((JSONArray) edgeInfo).toArray();
        LongWritable sinkID = new LongWritable(Long.valueOf(edgeValues[0].toString()));
        LongWritable edgeID = new LongWritable(Long.valueOf(edgeValues[1].toString()));
        //fix this
        //same format as vertex
        String[] eprop = edgeValues[2].toString().split(Pattern.quote("$"));
        MapWritable edgeMap = new MapWritable();
        for (int i = 0; i < eprop.length; i++) {
            String[] map = eprop[i].split(Pattern.quote(":"));
            Text key = new Text(map[0]);
            //FIXME:assuming String values for now
            Text value = new Text(map[2]);
            edgeMap.put(key, value);
        }

        Edge<E, LongWritable, LongWritable> edge = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID);
        E edgeValue = (E) edgeMap;
        edge.setValue(edgeValue);
        vertex.addEdge(edge);
    }
    return vertex;
}

---

From source file:io.aos.hdfs.MapWritableTest.java

License:Apache License

@Test
public void mapWritable() throws IOException {
    // vv MapWritableTest
    MapWritable src = new MapWritable();
    src.put(new IntWritable(1), new Text("cat"));
    src.put(new VIntWritable(2), new LongWritable(163));

    MapWritable dest = new MapWritable();
    WritableUtils.cloneInto(dest, src);/*  w w  w.  ja  v a2  s.c o  m*/
    assertThat((Text) dest.get(new IntWritable(1)), is(new Text("cat")));
    assertThat((LongWritable) dest.get(new VIntWritable(2)), is(new LongWritable(163)));
    // ^^ MapWritableTest
}

---

From source file:io.druid.indexer.InputRowSerde.java

License:Apache License

public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) {
    try {/* www .j  a v  a  2 s  . c o  m*/
        DataInput in = ByteStreams.newDataInput(data);

        //Read timestamp
        long timestamp = in.readLong();

        //Read dimensions
        StringArrayWritable sw = new StringArrayWritable();
        sw.readFields(in);
        List<String> dimensions = Arrays.asList(sw.toStrings());

        MapWritable mw = new MapWritable();
        mw.readFields(in);

        Map<String, Object> event = Maps.newHashMap();

        for (String d : dimensions) {
            Writable v = mw.get(new Text(d));

            if (v == null) {
                continue;
            }

            if (v instanceof Text) {
                event.put(d, ((Text) v).toString());
            } else if (v instanceof StringArrayWritable) {
                event.put(d, Arrays.asList(((StringArrayWritable) v).toStrings()));
            } else {
                throw new ISE("unknown dim value type %s", v.getClass().getName());
            }
        }

        //Read metrics
        for (AggregatorFactory aggFactory : aggs) {
            String k = aggFactory.getName();
            Writable v = mw.get(new Text(k));

            if (v == null) {
                continue;
            }

            String t = aggFactory.getTypeName();

            if (t.equals("float")) {
                event.put(k, ((FloatWritable) v).get());
            } else if (t.equals("long")) {
                event.put(k, ((LongWritable) v).get());
            } else {
                //its a complex metric
                ComplexMetricSerde serde = getComplexMetricSerde(t);
                BytesWritable bw = (BytesWritable) v;
                event.put(k, serde.fromBytes(bw.getBytes(), 0, bw.getLength()));
            }
        }

        return new MapBasedInputRow(timestamp, dimensions, event);
    } catch (IOException ex) {
        throw Throwables.propagate(ex);
    }
}

---

From source file:jadoop.HadoopGridJob.java

License:Open Source License

/**
 * Process the results that were returned by the Hadoop job. Each result
 * will be a key value pair with the format specified in the
 * HadoopGridTaskRunner class. The results for each key should be parsed and
 * placed into the HadoopGridTask object with the same key.
 * /*from   w w w .ja va 2s  .co m*/
 * @see HadoopGridTaskRunner
 * 
 * @throws IOException
 *             if there is a problem reading the results.
 */
private void processResults(FileSystem fs, Path outDir) throws IOException {

    FileStatus[] fileStatus = fs.listStatus(outDir);

    /*
     * Process the results for all of the tasks that have completed. Any
     * task that did not complete will be included in any file.
     */
    for (FileStatus file : fileStatus) {
        String fileName = file.getPath().getName();

        if (fileName.contains("part-m-")) {

            Path filePath = new Path(outDir + "/" + fileName);
            SequenceFile.Reader reader = new SequenceFile.Reader(job.getConfiguration(),
                    SequenceFile.Reader.file(filePath));

            Text mapperOutputKey = new Text();
            MapWritable mapperOutputVal = new MapWritable();

            /*
             * If multiple tasks are sent to the same node then the response
             * file will contain multiple entries. Be sure to process each
             * one of them.
             */
            while (reader.next(mapperOutputKey, mapperOutputVal)) {
                // Get the value returned from the HadoopGridTaskRunner.
                byte exitValue = ((ByteWritable) mapperOutputVal.get(new Text("EV"))).get();

                boolean taskTO = ((BooleanWritable) mapperOutputVal.get(new Text("TO"))).get();

                String stdOut = ((Text) mapperOutputVal.get(new Text("SO"))).toString();
                String stdErr = ((Text) mapperOutputVal.get(new Text("SE"))).toString();

                HadoopGridTask task = getTask(mapperOutputKey.toString());

                if (taskTO) {
                    task.markAsTimedout();
                } else {
                    // change the task's exit value.
                    task.markAsFinished(exitValue);
                }

                if (task.captureStandardOutput()) {
                    task.setStandardOutput(stdOut);
                }

                if (task.captureStandardError()) {
                    task.setStandardError(stdErr);
                }
            }

            reader.close();
        }
    }
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetails() throws Exception {
    deployTestIndices(1, 1);//from  w ww.j a  v  a 2s  .co  m
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit);
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNotNull(details.get(new Text("category")));
    }
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsWithFieldNames() throws Exception {
    deployTestIndices(1, 1);//from ww w .  java2s .  c o m
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    for (final Hit hit : hits.getHits()) {
        final MapWritable details = client.getDetails(hit, new String[] { "path" });
        final Set<Writable> keySet = details.keySet();
        assertFalse(keySet.isEmpty());
        assertNotNull(details.get(new Text("path")));
        assertNull(details.get(new Text("category")));
    }
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetBinaryDetails() throws Exception {
    File index = _temporaryFolder.newFolder("indexWithBinaryData");
    String textFieldName = "textField";
    String binaryFieldName = "binaryField";
    String textFieldContent = "sample text";
    byte[] bytesFieldContent = new byte[] { 1, 2, 3 };

    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(index), new StandardAnalyzer(Version.LUCENE_35),
            true, MaxFieldLength.UNLIMITED);
    Document document = new Document();
    document.add(new Field(binaryFieldName, bytesFieldContent, Store.YES));
    document.add(new Field(textFieldName, textFieldContent, Store.NO, Index.ANALYZED));
    indexWriter.addDocument(document);/* w  ww.j  a v  a  2  s.  co m*/
    indexWriter.optimize();
    indexWriter.close();
    DeployClient deployClient = new DeployClient(_miniCluster.getProtocol());
    IndexState indexState = deployClient.addIndex(index.getName(), index.getParentFile().getAbsolutePath(), 1)
            .joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer())
            .parse(textFieldName + ": " + textFieldContent);
    final Hits hits = client.search(query, new String[] { index.getName() }, 10);
    assertNotNull(hits);
    assertEquals(1, hits.getHits().size());
    final Hit hit = hits.getHits().get(0);
    final MapWritable details = client.getDetails(hit);
    final Set<Writable> keySet = details.keySet();
    assertEquals(1, keySet.size());
    final Writable writable = details.get(new Text(binaryFieldName));
    assertNotNull(writable);
    assertThat(writable, instanceOf(BytesWritable.class));
    BytesWritable bytesWritable = (BytesWritable) writable;
    bytesWritable.setCapacity(bytesWritable.getLength());// getBytes() returns
    // the full array
    assertArrayEquals(bytesFieldContent, bytesWritable.getBytes());
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testGetDetailsConcurrently() throws KattaException, ParseException, InterruptedException {
    deployTestIndices(1, 1);/* w w w  .j  a v  a2  s. c  om*/
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer()).parse("content: the");
    final Hits hits = client.search(query, new String[] { INDEX_NAME }, 10);
    assertNotNull(hits);
    assertEquals(10, hits.getHits().size());
    List<MapWritable> detailList = client.getDetails(hits.getHits());
    assertEquals(hits.getHits().size(), detailList.size());
    for (int i = 0; i < detailList.size(); i++) {
        final MapWritable details1 = client.getDetails(hits.getHits().get(i));
        final MapWritable details2 = detailList.get(i);
        assertEquals(details1.entrySet(), details2.entrySet());
        final Set<Writable> keySet = details2.keySet();
        assertFalse(keySet.isEmpty());
        final Writable writable = details2.get(new Text("path"));
        assertNotNull(writable);
    }
    client.close();
}

---

From source file:net.sf.katta.integrationTest.lib.lucene.LuceneClientTest.java

License:Apache License

@Test
public void testFilteredSearch() throws Exception {
    // write and deploy test index
    File filterIndex = _temporaryFolder.newFolder("filterIndex1");
    String textFieldName = "textField";
    String filterFieldName = "filterField";
    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(filterIndex),
            new StandardAnalyzer(Version.LUCENE_35), true, MaxFieldLength.UNLIMITED);
    for (int i = 0; i < 100; i++) {
        Document document = new Document();
        document.add(new Field(textFieldName, "sample " + i, Store.YES, Index.NOT_ANALYZED));
        document.add(new Field(filterFieldName, "" + (i % 10), Store.YES, Index.NOT_ANALYZED));
        indexWriter.addDocument(document);
    }/*from   w  w  w .j  a v a2  s . c  om*/
    indexWriter.optimize();
    indexWriter.close();
    DeployClient deployClient = new DeployClient(_miniCluster.getProtocol());
    IndexState indexState = deployClient
            .addIndex(filterIndex.getName(), filterIndex.getParentFile().getAbsolutePath(), 1).joinDeployment();
    assertEquals(IndexState.DEPLOYED, indexState);

    // build filter for terms in set {i | (i % 10) == 3}.
    ILuceneClient client = new LuceneClient(_miniCluster.getZkConfiguration());
    TermQuery filterQuery = new TermQuery(new Term(filterFieldName, "3"));
    QueryWrapperFilter filter = new QueryWrapperFilter(filterQuery);
    final Query query = new QueryParser(Version.LUCENE_35, "", new KeywordAnalyzer())
            .parse(textFieldName + ":" + "sample*3");

    final Hits hits = client.search(query, new String[] { filterIndex.getName() }, 100, null, filter);
    assertNotNull(hits);
    List<Hit> hitsList = hits.getHits();
    for (final Hit hit : hitsList) {
        writeToLog("hit", hit);
    }
    assertEquals(10, hits.size());
    assertEquals(10, hitsList.size());

    // check that returned results conform to the filter
    for (final Hit hit : hitsList) {
        MapWritable mw = client.getDetails(hit);
        Text text = (Text) mw.get(new Text("textField"));
        assertNotNull(text);
        String[] parts = text.toString().split(" ");
        assertTrue(parts.length == 2);
        int num = Integer.valueOf(parts[1]).intValue();
        assertTrue((num % 10) == 3);
    }

    client.close();
}

---

From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOElasticIT.java

License:Apache License

private String addFieldValuesToRow(String row, MapWritable mapw, String columnName) {
    Object valueObj = mapw.get(new Text(columnName));
    row += valueObj.toString() + "|";
    return row;//from  ww  w. j av  a2s  .co m
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »