Example usage for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value)

Source Link

Usage

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.deduplication.DocumentInfo.java

License:Apache License

public void setDocSimHash(LongWritable docSimHash) {
    this.docSimHash = new LongWritable(docSimHash.get());
}

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.helper.TextLongCountingReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable intWritable : values) {
        sum += intWritable.get();/* w ww. j  av a2s .  c om*/
    }

    context.write(key, new LongWritable(sum));
}

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.utils.URIExtractorTest.java

License:Apache License

@Test
public void testMapper() throws IOException, InterruptedException {
    final String expectedURI = "https://www.ukp.tu-darmstadt.de/ukp-home/";

    final WARCWritable warc = EasyMock.mock(WARCWritable.class);
    final WARCRecord record = EasyMock.mock(WARCRecord.class);
    final WARCRecord.Header header = EasyMock.mock(WARCRecord.Header.class);

    @SuppressWarnings("unchecked")
    final URIExtractor.URIExtractorMapper.Context context = EasyMock
            .mock(URIExtractor.URIExtractorMapper.Context.class);

    EasyMock.expect(record.getHeader()).andReturn(header);
    EasyMock.expect(warc.getRecord()).andReturn(record);
    EasyMock.expect(header.getTargetURI()).andReturn(expectedURI);
    context.write(new Text(expectedURI), NullWritable.get());
    EasyMock.replay(warc, record, header, context);

    final URIExtractor.URIExtractorMapper mapper = new URIExtractor.URIExtractorMapper();
    mapper.map(new LongWritable(0), warc, context);

    EasyMock.verify(warc, record, header, context);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * Moves a vertex from its old to its new partition.
 *
 * @param vertex           vertex//from w  ww .  j a va 2 s .  co m
 * @param desiredPartition partition to move vertex to
 */
private void migrateVertex(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex,
        long desiredPartition) {
    // add current partition to partition history
    vertex.getValue().addToPartitionHistory(vertex.getValue().getCurrentPartition().get());
    // decrease capacity in old partition
    String oldPartition = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get();
    notifyAggregator(oldPartition, NEGATIVE_ONE);
    // increase capacity in new partition
    String newPartition = CAPACITY_AGGREGATOR_PREFIX + desiredPartition;
    notifyAggregator(newPartition, POSITIVE_ONE);
    vertex.getValue().setCurrentPartition(new LongWritable(desiredPartition));
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * Initializes the vertex with a partition id. This is calculated using
 * modulo (vertex-id % number of partitions).
 *
 * @param vertex vertex//from  w  ww .jav a  2s  . c  o  m
 */
private void setVertexStartValue(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex) {
    long startValue = vertex.getId().get() % k;
    vertex.getValue().setCurrentPartition(new LongWritable(startValue));
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * The actual ADP computation./*from  ww  w . jav a2 s .c om*/
 *
 * @param vertex   Vertex
 * @param messages Messages that were sent to this vertex in the previous
 *                 superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<LongWritable, ARPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    if (getSuperstep() == 0) {
        if (!isPartitioned) {
            setVertexStartValue(vertex);
        }
        String aggregator = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get();
        notifyAggregator(aggregator, POSITIVE_ONE);
        sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition());
    } else {
        // even superstep: migrate phase
        if ((getSuperstep() % 2) == 0) {
            long desiredPartition = vertex.getValue().getDesiredPartition().get();
            long currentPartition = vertex.getValue().getCurrentPartition().get();
            long stableCounter = vertex.getValue().getStableCounter().get();
            if (desiredPartition != currentPartition) {
                boolean migrate = isAllowedToMigrate(desiredPartition);
                if (migrate) {
                    migrateVertex(vertex, desiredPartition);
                    sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition());
                    vertex.getValue().setStableCounter(new LongWritable(0));
                } else {
                    stableCounter++;
                    vertex.getValue().setStableCounter(new LongWritable(stableCounter));
                }
            }
            vertex.voteToHalt();
        } else { // odd superstep: demand phase
            if (vertex.getValue().getStableCounter().get() < stableThreshold) {
                long desiredPartition = getDesiredPartition(vertex, messages);
                vertex.getValue().setDesiredPartition(new LongWritable(desiredPartition));
                long currentValue = vertex.getValue().getCurrentPartition().get();
                boolean changed = currentValue != desiredPartition;
                if (changed) {
                    notifyAggregator(DEMAND_AGGREGATOR_PREFIX + desiredPartition, POSITIVE_ONE);
                }
            } else {
                vertex.voteToHalt();
            }
        }
    }
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get the current partition/*from  w ww .j  ava 2  s.co m*/
 *
 * @return the current partition
 */
public LongWritable getCurrentPartition() {
    return new LongWritable(this.currentPartition);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get method to get the desired partition
 *
 * @return the desired Partition//from ww  w. j  a  v a2 s . c o  m
 */
public LongWritable getDesiredPartition() {
    return new LongWritable(this.desiredPartition);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get the current stable counter/*  w w  w .  j av a  2  s. c  o  m*/
 *
 * @return the stable counter
 */
public LongWritable getStableCounter() {
    return new LongWritable(this.stableCounter);
}

From source file:de.unileipzig.dbs.giraph.algorithms.labelpropagation.LPComputation.java

License:Open Source License

/**
 * The actual LabelPropagation Computation
 *
 * @param vertex   Vertex/*from   ww w. j a  v  a  2  s .  c  om*/
 * @param messages Messages that were sent to this vertex in the previous
 *                 superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<LongWritable, LPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    if (getSuperstep() == 0) {
        sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity());
    } else {
        long currentCommunity = vertex.getValue().getCurrentCommunity().get();
        long lastCommunity = vertex.getValue().getLastCommunity().get();
        long newCommunity = getNewCommunity(vertex, messages);
        long currentStabilizationRound = vertex.getValue().getStabilizationRounds();

        // increment the stabilization count if vertex wants to stay in the
        // same partition
        if (lastCommunity == newCommunity) {
            currentStabilizationRound++;
            vertex.getValue().setStabilizationRounds(currentStabilizationRound);
        }

        boolean isUnstable = currentStabilizationRound <= stabilizationRounds;
        boolean mayChange = currentCommunity != newCommunity;
        if (mayChange && isUnstable) {
            vertex.getValue().setLastCommunity(new LongWritable(currentCommunity));
            vertex.getValue().setCurrentCommunity(new LongWritable(newCommunity));
            // reset stabilization counter
            vertex.getValue().setStabilizationRounds(0);
            sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity());
        }
    }
    vertex.voteToHalt();
}