Example usage for org.apache.hadoop.io DoubleWritable DoubleWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io DoubleWritable DoubleWritable.

Prototype

public DoubleWritable(double value) 

Source Link

Usage

From source file:org.apache.giraph.lib.TestTextDoubleDoubleAdjacencyListVertexInputFormat.java

License:Apache License

@Test
public void testLineSanitizer() throws Exception {
    String input = "Bye\t0.01\tCiao\t1.001\tTchau\t2.0001\tAdios\t3.00001";

    AdjacencyListVertexReader.LineSanitizer toUpper = new AdjacencyListVertexReader.LineSanitizer() {
        @Override/*w w w. j  a va 2s .  c  om*/
        public String sanitize(String s) {
            return s.toUpperCase();
        }
    };

    when(rr.getCurrentValue()).thenReturn(new Text(input));
    TextDoubleDoubleAdjacencyListVertexInputFormat.VertexReader<BooleanWritable> vr = new TextDoubleDoubleAdjacencyListVertexInputFormat.VertexReader<BooleanWritable>(
            rr, toUpper);

    vr.initialize(null, tac);
    assertTrue("Should have been able to read vertex", vr.nextVertex());
    BasicVertex<Text, DoubleWritable, DoubleWritable, BooleanWritable> vertex = vr.getCurrentVertex();
    setGraphState(vertex, graphState);
    assertValidVertex(conf, graphState, vertex, new Text("BYE"), new DoubleWritable(0.01d),
            new Edge<Text, DoubleWritable>(new Text("CIAO"), new DoubleWritable(1.001d)),
            new Edge<Text, DoubleWritable>(new Text("TCHAU"), new DoubleWritable(2.0001d)),
            new Edge<Text, DoubleWritable>(new Text("ADIOS"), new DoubleWritable(3.00001d)));

    assertEquals(vertex.getNumOutEdges(), 3);
}

---

From source file:org.apache.giraph.lib.TestTextDoubleDoubleAdjacencyListVertexInputFormat.java

License:Apache License

@Test
public void testDifferentSeparators() throws Exception {
    String input = "alpha:42:beta:99";

    when(rr.getCurrentValue()).thenReturn(new Text(input));
    conf.set(AdjacencyListVertexReader.LINE_TOKENIZE_VALUE, ":");
    TextDoubleDoubleAdjacencyListVertexInputFormat.VertexReader<BooleanWritable> vr = new TextDoubleDoubleAdjacencyListVertexInputFormat.VertexReader<BooleanWritable>(
            rr);// w  ww  . ja v a2s .c om

    vr.initialize(null, tac);
    assertTrue("Should have been able to read vertex", vr.nextVertex());
    BasicVertex<Text, DoubleWritable, DoubleWritable, BooleanWritable> vertex = vr.getCurrentVertex();
    setGraphState(vertex, graphState);
    assertValidVertex(conf, graphState, vertex, new Text("alpha"), new DoubleWritable(42d),
            new Edge<Text, DoubleWritable>(new Text("beta"), new DoubleWritable(99d)));
    assertEquals(vertex.getNumOutEdges(), 1);
}

---

From source file:org.apache.giraph.partition.TestGiraphTransferRegulator.java

License:Apache License

@Test
public void testGiraphTransferRegulator() {
    job.getConfiguration().setInt(GiraphTransferRegulator.MAX_VERTICES_PER_TRANSFER, 1);
    job.getConfiguration().setInt(GiraphTransferRegulator.MAX_EDGES_PER_TRANSFER, 3);
    OutEdges<IntWritable, DoubleWritable> edges = new ArrayListEdges<IntWritable, DoubleWritable>();
    edges.initialize(3);/*w w  w .  j  a v  a 2s .  co  m*/
    edges.add(EdgeFactory.create(new IntWritable(2), new DoubleWritable(22)));
    edges.add(EdgeFactory.create(new IntWritable(3), new DoubleWritable(33)));
    edges.add(EdgeFactory.create(new IntWritable(4), new DoubleWritable(44)));
    vertex.initialize(new IntWritable(1), new FloatWritable(1), edges);
    GiraphTransferRegulator gtr = new GiraphTransferRegulator(job.getConfiguration());
    PartitionOwner owner = mock(PartitionOwner.class);
    when(owner.getPartitionId()).thenReturn(57);
    assertFalse(gtr.transferThisPartition(owner));
    gtr.incrementCounters(owner, vertex);
    assertTrue(gtr.transferThisPartition(owner));
}

---

From source file:org.apache.giraph.ranking.LinkRank.LinkRankComputation.java

License:Apache License

/**
 * Receives messages from neighbors and sets new score.
 * @param vertex current vertex/* w w w.j a  v  a2  s.co m*/
 * @param messages messages from neighbors
 */
private void receiveMessages(Vertex<Text, DoubleWritable, NullWritable> vertex,
        Iterable<DoubleWritable> messages) {
    double sum = 0.0d;

    if (superStep == 0) {
        if (removeDuplicates) {
            removeDuplicateLinks(vertex);
        }
    } else if (1 <= superStep && superStep <= maxSteps - 4) {
        // find the score sum received from our neighbors.
        for (DoubleWritable message : messages) {
            sum += message.get();
        }

        Double newValue = ((1f - dampingFactor) / getTotalNumVertices())
                + dampingFactor * (sum + currentDanglingContribution);

        vertex.setValue(new DoubleWritable(newValue));
        LOG.info(vertex.getId().toString() + "=" + vertex.getValue().get());

    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.LinkRankComputation.java

License:Apache License

/**
 * Normalize the vertex scores./* w ww  . ja v  a  2 s.  co  m*/
 * Uses log scale and cumulative probability distribution.
 * @param vertex current vertex
 */
private void normalizeVertexScore(Vertex<Text, DoubleWritable, NullWritable> vertex) {

    double logValueDouble = Math.log(vertex.getValue().get());
    if (superStep == maxSteps - 4) {
        /**
         * Calculate LOG(value) and aggregate to SUM_OF_LOGS.
         */
        DoubleWritable logValue = new DoubleWritable(logValueDouble);
        aggregate(LinkRankComputation.SUM_OF_LOGS, logValue);
    } else if (superStep == maxSteps - 2) {
        /** Pass previous superstep since WorkerContext will need SUM_OF_LOGS
         *  to be aggregated.
         *  In this step, get AVG_OF_LOGS (calculated by WorkerContext)
         *  and calculate meanSquareError, aggregate it to SUM_OF_DEVS.
         *  WorkerContext will use SUM_OF_DEVS to calculate stdev
         *  in maxsupersteps-1 step.
         */
        double meanSquareError = Math.pow(logValueDouble - logAvg.get(), 2);
        DoubleWritable mseWritable = new DoubleWritable(meanSquareError);
        aggregate(LinkRankComputation.SUM_OF_DEVS, mseWritable);
    } else if (superStep == maxSteps) {
        /**
         * Pass maxsupersteps-1 step since WorkerContext will calculate stdev.
         * Use stdev and AVG_OF_LOGS to create a Normal Distribution.
         * Calculate CDF, scale it and set the new value.
         */
        double newValue = 1.0d;
        double stdevValue = stdev.get();
        if (stdevValue == 0.0d) {
            stdevValue = 1e-10;
        }

        NormalDistribution dist = new NormalDistributionImpl(logAvg.get(), stdevValue);
        try {
            double cumProb = dist.cumulativeProbability(logValueDouble);
            newValue = cumProb * scale;
        } catch (MathException e) {
            e.printStackTrace();
        }
        vertex.setValue(new DoubleWritable(newValue));
    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.LinkRankComputation.java

License:Apache License

/**
 * If we are at a superstep that is not the last one,
 * send messages to the neighbors./*from  w w w.j a  v a  2  s.  com*/
 * If it's the last step, vote to halt!
 *
 * @param vertex will distribute vertex's score to its neighbors.
 */
private void distributeScores(Vertex<Text, DoubleWritable, NullWritable> vertex) {
    int edgeCount = vertex.getNumEdges();

    if (superStep < maxSteps) {
        DoubleWritable message = new DoubleWritable(vertex.getValue().get() / edgeCount);

        if (edgeCount == 0) {
            aggregate(LinkRankComputation.DANGLING_AGG, vertex.getValue());
        } else {
            sendMessageToAllEdges(vertex, message);
        }
    } else {
        vertex.voteToHalt();
    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.LinkRankVertexMasterCompute.java

License:Apache License

@Override
public void compute() {
    // add additional 3 steps for normalization.
    long maxSteps = getContext().getConfiguration().getLong(LinkRankComputation.SUPERSTEP_COUNT, 10) + 3;
    long superstep = getSuperstep();
    if (superstep == maxSteps - 2) {
        /**/*  w  ww  .j  a v a  2s .co  m*/
         * We should have log values of the scores aggregated in SUM_OF_LOGS.
         * Divide this sum by total number of vertices and aggragate in
         * AVG_OF_LOGS.
         */
        DoubleWritable logsum = getAggregatedValue(LinkRankComputation.SUM_OF_LOGS);
        DoubleWritable avg = new DoubleWritable(logsum.get() / getTotalNumVertices());

        setAggregatedValue(LinkRankComputation.AVG_OF_LOGS, avg);

    } else if (superstep == maxSteps) {
        /**
         * Calculate standart deviation with deviation sums SUM_OF_DEVS.
         * Aggregate result to STDEV.
         */
        DoubleWritable devSum = getAggregatedValue(LinkRankComputation.SUM_OF_DEVS);
        double ratio = devSum.get() / getTotalNumVertices();
        DoubleWritable stdev = new DoubleWritable(Math.sqrt(ratio));
        setAggregatedValue(LinkRankComputation.STDEV, stdev);
    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.TrustRankComputation.java

License:Apache License

/**
 * Receives messages from neighbors and sets new score.
 * @param vertex current vertex/* w w w  .ja  v  a 2  s  .c o  m*/
 * @param messages messages from neighbors
 */
private void receiveMessages(Vertex<Text, DoubleWritable, NullWritable> vertex,
        Iterable<DoubleWritable> messages) {
    double sum = 0.0d;

    if (superStep == 0) {
        if (removeDuplicates) {
            removeDuplicateLinks(vertex);
        }
        if (Math.abs(vertex.getValue().get() - 1.0d) < 1e-3) {
            aggregate(TrustRankComputation.TRUSTED_VERTICES, new Text(";" + vertex.getId()));
            aggregate(TrustRankComputation.NUM_TRUSTED, new IntWritable(1));
        }

    } else if (1 <= superStep && superStep <= maxSteps - 4) {
        // find the score sum received from our neighbors.
        for (DoubleWritable message : messages) {
            sum += message.get();
        }

        HashSet<String> trusteds = new HashSet<String>();
        for (String element : getAggregatedValue(TrustRankComputation.TRUSTED_VERTICES).toString().split(";")) {
            trusteds.add(element);
        }

        if (!trusteds.contains(vertex.getId())) {
            currentDanglingContribution = 0.0d;
        }
        Double newValue = ((1f - dampingFactor) / getTotalNumVertices())
                + dampingFactor * (sum + currentDanglingContribution);

        vertex.setValue(new DoubleWritable(newValue));
        LOG.info(vertex.getId().toString() + "=" + vertex.getValue().get());

    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.TrustRankComputation.java

License:Apache License

/**
 * Normalize the vertex scores.//from   w  w w .j a  v a2s  .  c o m
 * Uses log scale and cumulative probability distribution.
 * @param vertex current vertex
 */
private void normalizeVertexScore(Vertex<Text, DoubleWritable, NullWritable> vertex) {

    double logValueDouble = Math.log(vertex.getValue().get());
    if (superStep == maxSteps - 4) {
        /**
         * Calculate LOG(value) and aggregate to SUM_OF_LOGS.
         */
        DoubleWritable logValue = new DoubleWritable(logValueDouble);
        aggregate(TrustRankComputation.SUM_OF_LOGS, logValue);
    } else if (superStep == maxSteps - 2) {
        /** Pass previous superstep since WorkerContext will need SUM_OF_LOGS
         *  to be aggregated.
         *  In this step, get AVG_OF_LOGS (calculated by WorkerContext)
         *  and calculate meanSquareError, aggregate it to SUM_OF_DEVS.
         *  WorkerContext will use SUM_OF_DEVS to calculate stdev
         *  in maxsupersteps-1 step.
         */
        double meanSquareError = Math.pow(logValueDouble - logAvg.get(), 2);
        DoubleWritable mseWritable = new DoubleWritable(meanSquareError);
        aggregate(TrustRankComputation.SUM_OF_DEVS, mseWritable);
    } else if (superStep == maxSteps) {
        /**
         * Pass maxsupersteps-1 step since WorkerContext will calculate stdev.
         * Use stdev and AVG_OF_LOGS to create a Normal Distribution.
         * Calculate CDF, scale it and set the new value.
         */
        double newValue = 1.0d;
        double stdevValue = stdev.get();
        if (stdevValue == 0.0d) {
            stdevValue = 1e-10;
        }

        NormalDistribution dist = new NormalDistributionImpl(logAvg.get(), stdevValue);
        try {
            double cdf = dist.cumulativeProbability(logValueDouble);
            newValue = cdf * scale;
        } catch (MathException e) {
            e.printStackTrace();
        }
        vertex.setValue(new DoubleWritable(newValue));
    }
}

---

From source file:org.apache.giraph.ranking.LinkRank.TrustRankComputation.java

License:Apache License

/**
 * If we are at a superstep that is not the last one,
 * send messages to the neighbors.//from  ww w . ja  v  a2  s . c o m
 * If it's the last step, vote to halt!
 *
 * @param vertex will distribute vertex's score to its neighbors.
 */
private void distributeScores(Vertex<Text, DoubleWritable, NullWritable> vertex) {
    int edgeCount = vertex.getNumEdges();

    if (superStep < maxSteps) {
        DoubleWritable message = new DoubleWritable(vertex.getValue().get() / edgeCount);

        if (edgeCount == 0) {
            aggregate(TrustRankComputation.DANGLING_AGG, vertex.getValue());
        } else {
            sendMessageToAllEdges(vertex, message);
        }
    } else {
        vertex.voteToHalt();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• »