Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:com.soteradefense.dga.io.formats.DGALongEdgeValueInputFormatTest.java

License:Apache License

@Test(expected = IOException.class)
public void testInputParserWithDelimiterInDataNoEscape() throws IOException, InterruptedException {
    String input = "te\tst@test.com\tanother@test.com\t10";
    when(rr.getCurrentValue()).thenReturn(new Text(input));
    EdgeReader ter = createEdgeReader(rr);
    conf.set(LINE_TOKENIZE_VALUE, "\t");
    ter.setConf(conf);//from  w  w w .  ja va  2s  .c om
    ter.initialize(null, tac);
    assertEquals(ter.getCurrentSourceId(), new Text("te\\tst@test.com"));
    assertEquals(ter.getCurrentEdge().getTargetVertexId(), new Text("another@test.com"));
    assertEquals(ter.getCurrentEdge().getValue(), new LongWritable(10L));

}

---

From source file:com.soteradefense.dga.io.formats.DGALongEdgeValueInputFormatTest.java

License:Apache License

@Test(expected = IOException.class)
public void testInputParserWithMalformedLine() throws IOException, InterruptedException {
    String input = "1";
    when(rr.getCurrentValue()).thenReturn(new Text(input));
    EdgeReader ter = createEdgeReader(rr);
    ter.setConf(conf);//w w w  . j  a v  a2  s  .co  m
    ter.initialize(null, tac);
    assertEquals(ter.getCurrentSourceId(), new Text("1"));
    assertEquals(ter.getCurrentEdge().getTargetVertexId(), new Text());
    assertEquals(ter.getCurrentEdge().getValue(), new LongWritable(1L));

}

---

From source file:com.soteradefense.dga.io.formats.DGALongEdgeValueInputFormatTest.java

License:Apache License

@Test(expected = IOException.class)
public void testInputParserWithMalformedLineAndDelimiter() throws IOException, InterruptedException {
    String input = "1,";
    when(rr.getCurrentValue()).thenReturn(new Text(input));
    EdgeReader ter = createEdgeReader(rr);
    ter.setConf(conf);/* ww w  .j  av  a2s  .  co m*/
    ter.initialize(null, tac);
    assertEquals(ter.getCurrentSourceId(), new Text("1"));
    assertEquals(ter.getCurrentEdge().getTargetVertexId(), new Text());
    assertEquals(ter.getCurrentEdge().getValue(), new LongWritable(1L));

}

---

From source file:com.soteradefense.dga.io.formats.DGALongEdgeValueInputFormatTest.java

License:Apache License

@Test(expected = IOException.class)
public void testInputParserWithMalformedLineAndDelimiterNoSource() throws IOException, InterruptedException {
    String input = ",1";
    when(rr.getCurrentValue()).thenReturn(new Text(input));
    EdgeReader ter = createEdgeReader(rr);
    ter.setConf(conf);//  w  w w  . j a  v  a2  s  . co  m
    ter.initialize(null, tac);
    assertEquals(ter.getCurrentSourceId(), new Text());
    assertEquals(ter.getCurrentEdge().getTargetVertexId(), new Text("1"));
    assertEquals(ter.getCurrentEdge().getValue(), new LongWritable(1L));

}

---

From source file:com.soteradefense.dga.louvain.giraph.LouvainComputation.java

License:Apache License

@Override
public void compute(Vertex<Text, LouvainNodeState, LongWritable> vertex, Iterable<LouvainMessage> messages)
        throws IOException {

    long currentSuperstep = getSuperstep();
    int currentMinorstep = (int) (currentSuperstep % 3); // the step in this iteration
    int currentIteration = (int) (currentSuperstep / 3); // the current iteration, two iterations make a full pass.

    //logger.info("currentSuperstep: " + currentSuperstep + " currentMinorstep: " + currentMinorstep + "currentIteration: " + currentIteration);

    LouvainNodeState vertexValue = vertex.getValue();

    // count the total edge weight of the graph on the first super step only
    if (currentSuperstep == 0) {
        if (!vertexValue.isFromLouvainVertexReader()) {
            vertexValue.setCommunity(vertex.getId().toString());
            long edgeWeightAggregation = 0;
            for (Edge<Text, LongWritable> edge : vertex.getEdges()) {
                edgeWeightAggregation += edge.getValue().get();
            }//from  w w w .ja  va 2s  .c  o  m
            vertexValue.setNodeWeight(edgeWeightAggregation);
        }
        aggregate(TOTAL_EDGE_WEIGHT_AGG,
                new LongWritable(vertexValue.getNodeWeight() + vertexValue.getInternalWeight()));
    } else if (vertexValue.getCommunity().equals("")) {
        vertexValue.setCommunity(vertex.getId().toString());
        vertexValue.setNodeWeight(0L);
    }

    if (currentSuperstep == 0 && vertex.getNumEdges() == 0) {
        // nodes that have no edges send themselves a message on the step 0
        this.sendMessage(vertex.getId(), new LouvainMessage());
        vertex.voteToHalt();
        return;
    } else if (currentSuperstep == 1 && vertex.getNumEdges() == 0) {
        // nodes that have no edges aggregate their Q value and exit computation on step 1
        double q = calculateActualQ(vertex, new ArrayList<LouvainMessage>());
        aggregateQ(q);
        vertex.voteToHalt();
        return;
    }

    // at the start of each full pass check to see if progress is still being made, if not halt
    if (currentMinorstep == 1 && currentIteration > 0 && currentIteration % 2 == 0) {
        vertexValue.setChanged(0L); // change count is per pass
        long totalChange = ((LongWritable) getAggregatedValue(CHANGE_AGG)).get();
        vertexValue.getChangeHistory().add(totalChange);

        // if halting aggregate q value and replace node edges with community edges (for next stage in pipeline)
        if (LouvainMasterCompute.decideToHalt(vertexValue.getChangeHistory(), getConf())) {
            double q = calculateActualQ(vertex, messages);
            replaceNodeEdgesWithCommunityEdges(vertex, messages);
            aggregateQ(q);
            return;
            // note: we did not vote to halt, MasterCompute will halt computation on next step
        }
    }

    try {
        switch (currentMinorstep) {
        case 0:
            getAndSendCommunityInfo(vertex, messages);

            // if the next step well require a progress check, aggregate the number of nodes who have changed community.
            if (currentIteration > 0 && currentIteration % 2 == 0) {
                aggregate(CHANGE_AGG, new LongWritable(vertexValue.getChanged()));
            }

            break;
        case 1:
            calculateBestCommunity(vertex, messages, currentIteration);
            break;
        case 2:
            updateCommunities(vertex, messages);
            break;
        default:
            throw new IllegalArgumentException("Invalid minorstep: " + currentMinorstep);
        }
    } finally {
        vertex.voteToHalt();
    }

}

---

From source file:com.soteradefense.dga.louvain.giraph.LouvainComputation.java

License:Apache License

/**
 * Replace each edge to a neighbor with an edge to that neigbors community
 * instead. Done just before exiting computation. In the next state of the
 * piple line this edges are aggregated and all communities are represented
 * as single nodes. Edges from the community to itself are tracked be the
 * ndoes interal weight.//from   w ww.j  a va2  s.co m
 *
 * @param messages
 */
private void replaceNodeEdgesWithCommunityEdges(Vertex<Text, LouvainNodeState, LongWritable> vertex,
        Iterable<LouvainMessage> messages) {

    // group messages by communities.
    HashMap<String, LouvainMessage> communityMap = new HashMap<String, LouvainMessage>();
    for (LouvainMessage message : messages) {

        String communityId = message.getCommunityId();

        if (communityMap.containsKey(communityId)) {
            LouvainMessage m = communityMap.get(communityId);
            m.setEdgeWeight(m.getEdgeWeight() + message.getEdgeWeight());
        } else {
            LouvainMessage newmess = new LouvainMessage(message);
            communityMap.put(communityId, newmess);
        }
    }

    List<Edge<Text, LongWritable>> edges = new ArrayList<Edge<Text, LongWritable>>(communityMap.size() + 1);
    for (Map.Entry<String, LouvainMessage> entry : communityMap.entrySet()) {
        edges.add(EdgeFactory.create(new Text(entry.getKey()),
                new LongWritable(entry.getValue().getEdgeWeight())));
    }
    vertex.setEdges(edges);
}

---

From source file:com.soteradefense.dga.louvain.giraph.LouvainTests.java

License:Apache License

private TestGraph<Text, LouvainNodeState, LongWritable> getGraph(GiraphConfiguration conf) {
    TestGraph<Text, LouvainNodeState, LongWritable> testGraph = new TestGraph<Text, LouvainNodeState, LongWritable>(
            conf);/* w  w  w .  ja  v a2  s.  c o m*/
    testGraph.addEdge(new Text("1"), new Text("2"), new LongWritable(1L));
    return testGraph;
}

---

From source file:com.soteradefense.dga.louvain.giraph.LouvainTests.java

License:Apache License

private TestGraph<Text, LouvainNodeState, LongWritable> getCyclicGraph(GiraphConfiguration conf) {
    TestGraph<Text, LouvainNodeState, LongWritable> testGraph = new TestGraph<Text, LouvainNodeState, LongWritable>(
            conf);//from  ww  w .  j a  va  2s  .  c o  m
    testGraph.addEdge(new Text("1"), new Text("2"), new LongWritable(1L));
    testGraph.addEdge(new Text("2"), new Text("1"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("3"), new LongWritable(1L));
    testGraph.addEdge(new Text("3"), new Text("1"), new LongWritable(1L));
    testGraph.addEdge(new Text("2"), new Text("3"), new LongWritable(1L));
    testGraph.addEdge(new Text("3"), new Text("2"), new LongWritable(1L));
    return testGraph;
}

---

From source file:com.soteradefense.dga.louvain.giraph.LouvainTests.java

License:Apache License

private TestGraph<Text, LouvainNodeState, LongWritable> getBiggerGraph(GiraphConfiguration conf) {
    TestGraph<Text, LouvainNodeState, LongWritable> testGraph = new TestGraph<Text, LouvainNodeState, LongWritable>(
            conf);/*from w w  w  .  ja v  a2s  .c o  m*/
    testGraph.addEdge(new Text("1"), new Text("2"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("3"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("4"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("5"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("6"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("7"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("8"), new LongWritable(1L));
    testGraph.addEdge(new Text("1"), new Text("9"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("10"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("11"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("12"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("13"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("14"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("15"), new LongWritable(1L));
    testGraph.addEdge(new Text("9"), new Text("16"), new LongWritable(1L));
    return testGraph;
}

---

From source file:com.splunk.shuttl.integration.hadoop.hbase.CSVMapperTest.java

License:Apache License

public void map_emptyText_DoesNotWriteAnythingToContext() throws IOException, InterruptedException {

    LongWritable offsetKey = new LongWritable(17);
    Text emptyCSVRow = new Text("");

    mapper.map(offsetKey, emptyCSVRow, context);

    verify(context, times(0)).write(any(), any());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »