Example usage for org.apache.hadoop.io DoubleWritable DoubleWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io DoubleWritable DoubleWritable.

Prototype

public DoubleWritable(double value) 

Source Link

Usage

From source file:main.okapi.cf.als.Als.java

License:Apache License

/**
 * Main ALS compute method.//from  w  ww.  ja  v  a2 s.  co  m
 * 
 * It updates the current latent vector based on ALS:<br>
 *  A = M * M^T + LAMBDA * N * E<br>
 *  V = M * R<br>
 *  A * U = V, then solve for U<br>
 *  <br> 
 *  where<br>
 *  R: column vector with ratings by this user<br>
 *  M: item features for items rated by this user with dimensions |F|x|R|<br>
 *  M^T: transpose of M with dimensions |R|x|F|<br>
 *  N: number of ratings of this user<br>
 *  E: identity matrix with dimensions |F|x|F|<br>
 * 
 * @param messages Messages received
 */
public final void compute(Vertex<CfLongId, FloatMatrixWritable, FloatWritable> vertex,
        final Iterable<FloatMatrixMessage> messages) {

    FloatMatrix mat_M = new FloatMatrix(vectorSize, vertex.getNumEdges());
    FloatMatrix mat_R = new FloatMatrix(vertex.getNumEdges(), 1);

    // Build the matrices of the linear system
    int i = 0;
    for (FloatMatrixMessage msg : messages) {
        mat_M.putColumn(i, msg.getFactors());
        mat_R.put(i, 0, vertex.getEdgeValue(msg.getSenderId()).get());
        i++;
    }

    updateValue(vertex.getValue(), mat_M, mat_R, lambda);

    // Calculate errors and add squares to the RMSE aggregator
    double rmsePartialSum = 0d;
    for (int j = 0; j < mat_M.columns; j++) {
        float prediction = vertex.getValue().dot(mat_M.getColumn(j));
        double error = prediction - mat_R.get(j, 0);
        rmsePartialSum += (error * error);
    }

    aggregate(RMSE_AGGREGATOR, new DoubleWritable(rmsePartialSum));

    // Propagate new value
    sendMessageToAllEdges(vertex, new FloatMatrixMessage(vertex.getId(), vertex.getValue(), 0.0f));

    vertex.voteToHalt();
}

---

From source file:main.okapi.cf.sgd.Sgd.java

License:Apache License

/**
 * Main SGD compute method./*ww w.  j a v a  2  s  . c  o  m*/
 * 
 * @param messages
 *          Messages received
 */
public final void compute(Vertex<CfLongId, FloatMatrixWritable, FloatWritable> vertex,
        final Iterable<FloatMatrixMessage> messages) {

    double rmsePartialSum = 0d;
    float l2norm = 0f;

    if (tolerance > 0) {
        // Create new object because we're going to operate on the old one.
        oldValue = new FloatMatrixWritable(vertex.getValue().getRows(), vertex.getValue().getColumns(),
                vertex.getValue().data);
    }

    for (FloatMatrixMessage msg : messages) {
        // Get rating for the item that this message came from
        float rating = vertex.getEdgeValue(msg.getSenderId()).get();

        // Update the factors,
        // this process do exactly what Stochastic Gradient Descent do.
        // Iterate samples and update parameters one at a time.
        updateValue(vertex.getValue(), msg.getFactors(), rating, minRating, maxRating, lambda, gamma);
    }

    // Calculate new error for RMSE calculation
    for (FloatMatrixMessage msg : messages) {
        float predicted = vertex.getValue().dot(msg.getFactors());
        float rating = vertex.getEdgeValue(msg.getSenderId()).get();
        predicted = Math.min(predicted, maxRating);
        predicted = Math.max(predicted, minRating);
        float err = predicted - rating;
        rmsePartialSum += (err * err);
    }

    aggregate(RMSE_AGGREGATOR, new DoubleWritable(rmsePartialSum));

    // Calculate difference with previous value
    if (tolerance > 0) {
        l2norm = vertex.getValue().distance2(oldValue);
    }

    // Broadcast the new vector
    if (tolerance < 0 || (tolerance > 0 && l2norm > tolerance)) {
        sendMessageToAllEdges(vertex, new FloatMatrixMessage(vertex.getId(), vertex.getValue(), 0.0f));
    }

    vertex.voteToHalt();
}

---

From source file:main.okapi.examples.SimpleCountInEdgesVertex.java

License:Apache License

/**
 * Compute method/*w ww . ja  va2 s. c o m*/
 * @param messages Messages received
 */
public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex,
        Iterable<DoubleWritable> messages) {
    /** Initialize vertex value to zero */
    if (getSuperstep() == 0) {
        vertex.setValue(new DoubleWritable(0d));
    }
    /** Initialize counter */
    double count = 0d;
    /** Count messages */
    for (@SuppressWarnings("unused")
    DoubleWritable message : messages) {
        count += 1d;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Vertex " + vertex.getId() + " got a message. New total = " + count);
    }
    /** Save count into vertex value */
    vertex.setValue(new DoubleWritable(count));

    /** Send to all neighbors a message */
    for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Vertex " + vertex.getId() + " sent a message to " + edge.getTargetVertexId());
        }
        if (getSuperstep() < 2) {
            sendMessage(edge.getTargetVertexId(), new DoubleWritable(1d));
        }
    }
    vertex.voteToHalt();
}

---

From source file:main.okapi.examples.SimpleMasterComputeVertex.java

License:Apache License

@Override
public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex,
        Iterable<DoubleWritable> messages) {
    double oldSum = getSuperstep() == 0 ? 0 : vertex.getValue().get();
    double newValue = this.<DoubleWritable>getAggregatedValue(SMC_AGG).get();
    double newSum = oldSum + newValue;
    vertex.setValue(new DoubleWritable(newSum));

    SimpleMasterComputeWorkerContext workerContext = (SimpleMasterComputeWorkerContext) getWorkerContext();
    workerContext.setFinalSum(newSum);//  w w w  .  j a  va  2 s.c o m
    LOG.info("Current sum: " + newSum);
}

---

From source file:main.okapi.graphs.SimplePageRank.java

License:Apache License

@Override
public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex,
        Iterable<DoubleWritable> messages) {
    if (getSuperstep() == 0) {
        vertex.setValue(new DoubleWritable(1f / getTotalNumVertices()));
    }//  www . j  a va2 s  .c  o m
    if (getSuperstep() >= 1) {
        double sum = 0;
        for (DoubleWritable message : messages) {
            sum += message.get();
        }
        DoubleWritable vertexValue = new DoubleWritable((0.15f / getTotalNumVertices()) + 0.85f * sum);
        vertex.setValue(vertexValue);
    }

    if (getSuperstep() < getContext().getConfiguration().getInt(MAX_SUPERSTEPS, MAX_SUPERSTEPS_DEFAULT)) {

        long edges = vertex.getNumEdges();
        sendMessageToAllEdges(vertex, new DoubleWritable(vertex.getValue().get() / edges));
    } else {
        vertex.voteToHalt();
    }
}

---

From source file:main.okapi.graphs.SingleSourceShortestPaths.java

License:Apache License

@Override
public void compute(Vertex<LongWritable, DoubleWritable, FloatWritable> vertex,
        Iterable<DoubleWritable> messages) {
    if (getSuperstep() == 0) {
        vertex.setValue(new DoubleWritable(Double.MAX_VALUE));
    }/*from  w ww. j av a 2 s  .  c  o  m*/

    // In directed graphs, vertices that have no outgoing edges will be created
    // in the 1st superstep as a result of messages sent to them.
    if (getSuperstep() == 1 && vertex.getNumEdges() == 0) {
        vertex.setValue(new DoubleWritable(Double.MAX_VALUE));
    }

    double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE;
    for (DoubleWritable message : messages) {
        minDist = Math.min(minDist, message.get());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Vertex " + vertex.getId() + " got minDist = " + minDist + " vertex value = "
                + vertex.getValue());
    }
    if (minDist < vertex.getValue().get()) {
        vertex.setValue(new DoubleWritable(minDist));
        for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) {
            double distance = minDist + edge.getValue().get();
            if (LOG.isDebugEnabled()) {
                LOG.debug(
                        "Vertex " + vertex.getId() + " sent to " + edge.getTargetVertexId() + " = " + distance);
            }
            sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance));
        }
    }
    vertex.voteToHalt();
}

---

From source file:ml.grafos.okapi.cf.sgd.Sgd.java

License:Apache License

/**
 * Main SGD compute method./*from ww  w. ja v a 2  s . com*/
 * 
 * @param messages
 *          Messages received
 */
public final void compute(Vertex<CfLongId, FloatMatrixWritable, FloatWritable> vertex,
        final Iterable<FloatMatrixMessage> messages) {

    double rmsePartialSum = 0d;
    float l2norm = 0f;

    if (tolerance > 0) {
        // Create new object because we're going to operate on the old one.
        oldValue = new FloatMatrixWritable(vertex.getValue().getRows(), vertex.getValue().getColumns(),
                vertex.getValue().data);
    }

    for (FloatMatrixMessage msg : messages) {
        // Get rating for the item that this message came from
        float rating = vertex.getEdgeValue(msg.getSenderId()).get();

        // Update the factors
        updateValue(vertex.getValue(), msg.getFactors(), rating, minRating, maxRating, lambda, gamma);
    }

    // Calculate new error for RMSE calculation
    for (FloatMatrixMessage msg : messages) {
        float predicted = vertex.getValue().dot(msg.getFactors());
        float rating = vertex.getEdgeValue(msg.getSenderId()).get();
        predicted = Math.min(predicted, maxRating);
        predicted = Math.max(predicted, minRating);
        float err = predicted - rating;
        rmsePartialSum += (err * err);
    }

    aggregate(RMSE_AGGREGATOR, new DoubleWritable(rmsePartialSum));

    // Calculate difference with previous value
    if (tolerance > 0) {
        l2norm = vertex.getValue().distance2(oldValue);
    }

    // Broadcast the new vector
    if (tolerance < 0 || (tolerance > 0 && l2norm > tolerance)) {
        sendMessageToAllEdges(vertex, new FloatMatrixMessage(vertex.getId(), vertex.getValue(), 0.0f));
    }

    vertex.voteToHalt();
}

---

From source file:ml.grafos.okapi.clustering.ap.AffinityPropagation.java

License:Apache License

@Override
public void compute(Vertex<APVertexID, APVertexValue, DoubleWritable> vertex, Iterable<APMessage> messages)
        throws IOException {
    logger.trace("vertex {}, superstep {}", vertex.getId(), getSuperstep());

    final int maxIter = getContext().getConfiguration().getInt(MAX_ITERATIONS, MAX_ITERATIONS_DEFAULT);

    // Phases of the algorithm

    for (APMessage message : messages) {
        vertex.getValue().lastReceivedMessages.put(new APVertexID(message.from),
                new DoubleWritable(message.value));
    }//from  www .  j a  va 2  s. co m

    if (this.<LongWritable>getAggregatedValue("converged").get() == getTotalNumVertices()) {
        if (!vertex.getValue().exemplarCalc.get()) {
            computeExemplars(vertex);
            vertex.getValue().exemplarCalc.set(true);
        } else {
            computeClusters(vertex);
        }
    } else {
        if (getSuperstep() == 0) {
            initRows(vertex);
        } else if (getSuperstep() == 1) {
            initColumns(vertex, messages);
        } else if (getSuperstep() < maxIter) {
            computeBMSIteration(vertex);
        } else if (getSuperstep() == maxIter) {
            computeExemplars(vertex);
        } else {
            computeClusters(vertex);
        }
    }
}

---

From source file:ml.grafos.okapi.clustering.ap.AffinityPropagation.java

License:Apache License

private void initRowsFromVertexInput(Vertex<APVertexID, APVertexValue, DoubleWritable> vertex) {
    final long nVertices = getTotalNumVertices();
    for (int i = 1; i <= nVertices; i++) {
        APVertexID neighbor = new APVertexID(APVertexType.E, i);
        vertex.getValue().lastSentMessages.put(neighbor, new DoubleWritable(0));
        vertex.getValue().lastReceivedMessages.put(neighbor, new DoubleWritable(0));
        sendMessage(neighbor, new APMessage(vertex.getId(), 0));
        logger.trace("Init rows: {} -> {} : {}", vertex.getId(), neighbor, 0);
    }// w ww .  j  a v  a  2 s. c om
}

---

From source file:ml.grafos.okapi.clustering.ap.AffinityPropagation.java

License:Apache License

private void initRowsFromEdgeInput(Vertex<APVertexID, APVertexValue, DoubleWritable> vertex)
        throws IOException {
    for (Edge<APVertexID, DoubleWritable> edge : vertex.getEdges()) {
        APVertexID neighbor = new APVertexID(edge.getTargetVertexId());
        DoubleWritable weight = new DoubleWritable(edge.getValue().get());
        vertex.getValue().weights.put(neighbor, weight);
        vertex.getValue().lastSentMessages.put(neighbor, new DoubleWritable(0));
        vertex.getValue().lastReceivedMessages.put(neighbor, new DoubleWritable(0));
        sendMessage(neighbor, new APMessage(vertex.getId(), 0));
        logger.trace("Init rows:{} -> {} : {}", vertex.getId(), neighbor, 0);
        vertex.removeEdges(neighbor);//from   www .j  av  a 2 s. c  om
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• »