Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get() 

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:nl.tudelft.graphalytics.giraph.algorithms.wcc.DirectedWeaklyConnectedComponentsComputation.java

License:Apache License

/**
 * Propagates the smallest vertex id to all neighbors. Will always choose to
 * halt and only reactivate if a smaller id has been sent to it.
 *//*from   ww w  .  j a  v a2  s .c om*/
@Override
public void compute(Vertex<LongWritable, LongWritable, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    // Weakly connected components algorithm treats a directed graph as undirected, so we create the missing edges
    if (getSuperstep() == 0) {
        // Broadcast own id to notify neighbours of incoming edge
        sendMessageToAllEdges(vertex, vertex.getId());
    } else if (getSuperstep() == 1) {
        // For every incoming edge that does not have a corresponding outgoing edge, create one
        edgeSet.clear();
        for (Edge<LongWritable, NullWritable> existingEdge : vertex.getEdges()) {
            edgeSet.add(existingEdge.getTargetVertexId().get());
        }
        for (LongWritable incomingId : messages) {
            if (!edgeSet.contains(incomingId.get())) {
                vertex.addEdge(EdgeFactory.create(incomingId));
            }
        }

        // Initialize value to minimum id of neighbours
        long minId = vertex.getId().get();
        for (Edge<LongWritable, NullWritable> edge : vertex.getEdges()) {
            long targetVertexId = edge.getTargetVertexId().get();
            if (targetVertexId < minId) {
                minId = targetVertexId;
            }
        }

        // Store the new component id and broadcast it if it is not equal to this vertex's own id
        vertex.getValue().set(minId);
        if (minId != vertex.getId().get()) {
            sendMessageToAllEdges(vertex, vertex.getValue());
        }

        vertex.voteToHalt();
    } else {
        long currentComponent = vertex.getValue().get();

        // did we get a smaller id ?
        for (LongWritable message : messages) {
            long candidateComponent = message.get();
            if (candidateComponent < currentComponent) {
                currentComponent = candidateComponent;
            }
        }

        // propagate new component id to the neighbors
        if (currentComponent != vertex.getValue().get()) {
            vertex.getValue().set(currentComponent);
            sendMessageToAllEdges(vertex, vertex.getValue());
        }

        vertex.voteToHalt();
    }
}

---

From source file:nl.tudelft.graphalytics.giraph.algorithms.wcc.UndirectedWeaklyConnectedComponentsComputation.java

License:Apache License

/**
 * Propagates the smallest vertex id to all neighbors. Will always choose to
 * halt and only reactivate if a smaller id has been sent to it.
 */// w  w  w . j  av  a2 s .  c  om
@Override
public void compute(Vertex<LongWritable, LongWritable, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    // First superstep is special, because we can simply look at the neighbors
    if (getSuperstep() == 0) {
        // Initialize value to minimum id of neighbours
        long minId = vertex.getId().get();
        for (Edge<LongWritable, NullWritable> edge : vertex.getEdges()) {
            long targetVertexId = edge.getTargetVertexId().get();
            if (targetVertexId < minId) {
                minId = targetVertexId;
            }
        }

        // Store the new component id and broadcast it if it is not equal to this vertex's own id
        vertex.getValue().set(minId);
        if (minId != vertex.getId().get()) {
            sendMessageToAllEdges(vertex, vertex.getValue());
        }

        vertex.voteToHalt();
    } else {
        long currentComponent = vertex.getValue().get();

        // did we get a smaller id ?
        for (LongWritable message : messages) {
            long candidateComponent = message.get();
            if (candidateComponent < currentComponent) {
                currentComponent = candidateComponent;
            }
        }

        // propagate new component id to the neighbors
        if (currentComponent != vertex.getValue().get()) {
            vertex.getValue().set(currentComponent);
            sendMessageToAllEdges(vertex, vertex.getValue());
        }

        vertex.voteToHalt();
    }
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.conversion.DirectedVertexMapper.java

License:Apache License

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    // Parse the line (value) as a list of longs: vertex-id neighbour-1 neighbour-2 ...
    String[] vertices = whitespacePattern.split(value.toString());
    if (vertices.length == 0) {
        context.getCounter(Counters.ParseErrors.INVALID_LINE_FORMAT).increment(1);
        return;//from   w w  w  . j  a v  a2 s  .c o  m
    } else if (vertices.length == 1) {
        long vertexId = Long.parseLong(vertices[0]);
        context.write(new LongWritable(vertexId), new EdgeData(vertexId, true));
        return;
    }

    LongWritable sourceId;
    LongWritable destinationId;
    try {
        // Loop through the neighbour IDs and output an edge both ways for each
        sourceId = new LongWritable(Long.parseLong(vertices[0]));
        for (int i = 1; i < vertices.length; i++) {
            destinationId = new LongWritable(Long.parseLong(vertices[i]));
            context.write(sourceId, new EdgeData(destinationId.get(), true));
            context.write(destinationId, new EdgeData(sourceId.get(), false));
        }
    } catch (NumberFormatException ex) {
        context.getCounter(Counters.ParseErrors.NUMBER_FORMAT_EXCEPTION).increment(1);
        return;
    }
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.conversion.DirectedVertexReducer.java

License:Apache License

@Override
protected void reduce(LongWritable key, Iterable<EdgeData> values, Context context)
        throws IOException, InterruptedException {
    // Fill separate buffers for incoming and outgoing edges
    StringBuffer sbIn = new StringBuffer();
    StringBuffer sbOut = new StringBuffer();

    // Loop through the messages and add them to the buffers
    boolean foundIn = false, foundOut = false;
    for (EdgeData edge : values) {
        if (edge.getTargetId() == key.get()) {
            // Ignore, this self-edge was added to force this vertex's existence
        } else if (edge.isOutgoing()) {
            if (foundOut)
                sbOut.append(',');
            sbOut.append(edge.getTargetId());
            foundOut = true;/*from ww  w  . jav a  2 s. c o m*/
        } else {
            if (foundIn)
                sbIn.append(',');
            sbIn.append(edge.getTargetId());
            foundIn = true;
        }
    }

    // Combine the vertex ID and neighbour lists using Marcin's format
    StringBuffer out = new StringBuffer(key.toString());
    out.append("\t#").append(sbIn.toString()).append("\t@").append(sbOut.toString());
    if (!foundOut)
        out.append('\t');

    // Output the constructed line
    outValue.set(out.toString());
    context.write(NullWritable.get(), outValue);
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.conversion.UndirectedVertexReducer.java

License:Apache License

@Override
protected void reduce(LongWritable key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    // Combine the vertex ID and neighbour IDs using a StringBuilder
    StringBuilder sb = new StringBuilder();
    sb.append(key.get());
    for (LongWritable neighbour : values) {
        sb.append(' ').append(neighbour.get());
    }/*from www  .ja va2s  . c  o  m*/

    // Output the constructed line
    outValue.set(sb.toString());
    context.write(NullWritable.get(), outValue);
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.evo.DirectedForestFireModelMap.java

License:Apache License

public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter)
        throws IOException {
    DirectedNode node = new DirectedNode();
    node.readFields(value.toString());/*  ww w  .ja va2 s .  c o  m*/

    if (this.isFirst) { // INIT_JOB
        this.isFirst = false;
        // create N new vertices
        for (int i = 0; i < this.newVerticesPerSlot; i++) {
            long newID = this.taskID * this.newVerticesPerSlot + i + this.maxID;
            DirectedNode newVertex = new DirectedNode(String.valueOf(newID), new Vector<Edge>(),
                    new Vector<Edge>());

            this.newVertices.add(new LongWritable(newID)); // same as in Giraph can connect only to worker ambassadors

            oKey.set(newID);
            oVal.set(newVertex.toText());
            output.collect(oKey, oVal);
        }
    } else if (this.ambassadors.containsKey(new LongWritable(Long.parseLong(node.getId())))) { //update vertex
        Vector<Edge> edges = node.getInEdges();

        for (LongWritable id : this.ambassadors.get(new LongWritable(Long.parseLong(node.getId()))))
            edges.add(new Edge(node.getId(), id.toString()));
        node.setInEdges(edges);
    } else if (Long.parseLong(node.getId()) < this.maxID) { // check if potential ambassador n send to new vertex
        Set<LongWritable> edges = new HashSet<LongWritable>();
        for (Edge out : node.getOutEdges())
            edges.add(new LongWritable(Long.parseLong(out.getDest())));
        for (Edge in : node.getInEdges())
            edges.add(new LongWritable(Long.parseLong(in.getSrc())));

        for (LongWritable neighbour : edges) {
            if (ambassadors.containsKey(neighbour)) {
                // send my id to new vertices
                List<LongWritable> newVertices = this.ambassadors.get(neighbour);
                for (LongWritable id : newVertices)
                    output.collect(id, new Text(node.getId()));
            }
        }
    }

    // Init step -> pass all worker verticesIDs to all newVertices from this worker
    if (this.isInit) {
        oVal.set(node.getId());
        for (LongWritable id : this.newVertices) {
            oKey.set(id.get());
            output.collect(oKey, oVal);
        }
    }

    // pass node
    oKey.set(Long.parseLong(node.getId()));
    oVal.set(node.toText());
    output.collect(oKey, oVal);
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.evo.UndirectedForestFireModelMap.java

License:Apache License

public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter)
        throws IOException {
    UndirectedNode node = new UndirectedNode();
    node.readFields(value.toString());//from  w w w.java  2 s  . c  om

    if (this.isFirst) { // INIT_JOB
        this.isFirst = false;
        // create N new vertices
        for (int i = 0; i < this.newVerticesPerSlot; i++) {
            long newID = this.taskID * this.newVerticesPerSlot + i + this.maxID;
            UndirectedNode newVertex = new UndirectedNode(String.valueOf(newID), new Vector<Edge>());

            this.newVertices.add(new LongWritable(newID)); // same as in Giraph can connect only to worker ambassadors

            oKey.set(newID);
            oVal.set(newVertex.toText());
            output.collect(oKey, oVal);
        }
    } else if (this.ambassadors.containsKey(new LongWritable(Long.parseLong(node.getId())))) { //update vertex
        Vector<Edge> edges = node.getEdges();

        for (LongWritable id : this.ambassadors.get(new LongWritable(Long.parseLong(node.getId()))))
            edges.add(new Edge(node.getId(), id.toString()));
        node.setEdges(edges);
    } else if (Long.parseLong(node.getId()) < this.maxID) { // check if potential ambassador n send to new vertex
        for (Edge edge : node.getEdges()) {
            long neighbour = Long.parseLong(edge.getDest());
            if (ambassadors.containsKey(new LongWritable(neighbour))) {
                // send my id to new vertices
                List<LongWritable> newVertices = this.ambassadors.get(new LongWritable(neighbour));
                for (LongWritable id : newVertices)
                    output.collect(id, new Text(node.getId()));
            }
        }
    }

    // Init step -> pass all worker verticesIDs to all newVertices from this worker
    if (this.isInit) {
        oVal.set(node.getId());
        for (LongWritable id : this.newVertices) {
            oKey.set(id.get());
            output.collect(oKey, oVal);
        }
    }

    // pass node
    oKey.set(Long.parseLong(node.getId()));
    oVal.set(node.toText());
    output.collect(oKey, oVal);
}

---

From source file:nl.tudelft.graphalytics.mapreducev2.evo.UndirectedForestFireModelReducer.java

License:Apache License

private boolean processMsgs(LongWritable key, Iterator<Text> iterator,
        OutputCollector<NullWritable, Text> output) throws IOException {
    boolean result = false;

    while (iterator.hasNext()) {
        String value = iterator.next().toString();
        String[] data = value.split("\t");

        // new Vertex
        if (Long.parseLong(data[0]) >= this.maxID && data.length > 1) {
            result = true;/*from   www  . j a  va  2s. c o m*/
            this.newVertex.readFields(value);
        } else if (Long.parseLong(data[0]) >= this.maxID && this.isInit) {
            result = true;
            this.newVertex.readFields(value);
        } else {
            if (data.length > 1 || key.get() < this.maxID) { // passing vertex
                UndirectedNode passingVertex = new UndirectedNode();
                passingVertex.readFields(value);
                oVal.set(passingVertex.toText());
                output.collect(null, passingVertex.toText());
            } else { // potential ambassador
                potentialAmbassadors.add(Long.parseLong(value.trim()));
            }
        }
    }

    return result;
}

---

From source file:org.acacia.csr.java.SortComparator.java

License:Apache License

@Override
public int compare(WritableComparable a, WritableComparable b) {
    LongWritable v1 = (LongWritable) a;
    LongWritable v2 = (LongWritable) b;/*from ww w. j a  v a2 s .c  om*/

    if (v1.get() > v2.get()) {
        return 1;
    } else if (v1.get() < v2.get()) {
        return -1;
    } else {
        return 0;
    }
}

---

From source file:org.acacia.csr.java.VertexPartitioner.java

License:Apache License

public int getPartition(LongWritable key, Text values, int numReduceTasks) {
    int part = 0;
    long k = key.get();
    int vertsPerPart = -1;

    if (numReduceTasks > 1) {
        vertsPerPart = (int) (numberOfVerts / (numReduceTasks - 1));

        if (vertsPerPart != 0) {
            part = (int) (k / vertsPerPart);
        }/*from  www .  jav  a 2s  .  co m*/
    }

    //System.out.println("vertex : " + k + " part : " + part);

    return part;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »