Example usage for org.apache.hadoop.io LongWritable LongWritable

List of usage examples for org.apache.hadoop.io LongWritable LongWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable LongWritable.

Prototype

public LongWritable(long value) 

Source Link

Usage

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.deduplication.DocumentInfo.java

License:Apache License

public void setDocSimHash(LongWritable docSimHash) {
    this.docSimHash = new LongWritable(docSimHash.get());
}

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.helper.TextLongCountingReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable intWritable : values) {
        sum += intWritable.get();/* w ww. j  av a2s .  c om*/
    }

    context.write(key, new LongWritable(sum));
}

From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.utils.URIExtractorTest.java

License:Apache License

@Test
public void testMapper() throws IOException, InterruptedException {
    final String expectedURI = "https://www.ukp.tu-darmstadt.de/ukp-home/";

    final WARCWritable warc = EasyMock.mock(WARCWritable.class);
    final WARCRecord record = EasyMock.mock(WARCRecord.class);
    final WARCRecord.Header header = EasyMock.mock(WARCRecord.Header.class);

    @SuppressWarnings("unchecked")
    final URIExtractor.URIExtractorMapper.Context context = EasyMock
            .mock(URIExtractor.URIExtractorMapper.Context.class);

    EasyMock.expect(record.getHeader()).andReturn(header);
    EasyMock.expect(warc.getRecord()).andReturn(record);
    EasyMock.expect(header.getTargetURI()).andReturn(expectedURI);
    context.write(new Text(expectedURI), NullWritable.get());
    EasyMock.replay(warc, record, header, context);

    final URIExtractor.URIExtractorMapper mapper = new URIExtractor.URIExtractorMapper();
    mapper.map(new LongWritable(0), warc, context);

    EasyMock.verify(warc, record, header, context);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * Moves a vertex from its old to its new partition.
 *
 * @param vertex           vertex//from w  ww .  j a va 2 s .  co m
 * @param desiredPartition partition to move vertex to
 */
private void migrateVertex(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex,
        long desiredPartition) {
    // add current partition to partition history
    vertex.getValue().addToPartitionHistory(vertex.getValue().getCurrentPartition().get());
    // decrease capacity in old partition
    String oldPartition = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get();
    notifyAggregator(oldPartition, NEGATIVE_ONE);
    // increase capacity in new partition
    String newPartition = CAPACITY_AGGREGATOR_PREFIX + desiredPartition;
    notifyAggregator(newPartition, POSITIVE_ONE);
    vertex.getValue().setCurrentPartition(new LongWritable(desiredPartition));
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * Initializes the vertex with a partition id. This is calculated using
 * modulo (vertex-id % number of partitions).
 *
 * @param vertex vertex//from  w  ww .jav a  2s  . c  o  m
 */
private void setVertexStartValue(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex) {
    long startValue = vertex.getId().get() % k;
    vertex.getValue().setCurrentPartition(new LongWritable(startValue));
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java

License:Open Source License

/**
 * The actual ADP computation./*from  ww  w . jav a2 s .c om*/
 *
 * @param vertex   Vertex
 * @param messages Messages that were sent to this vertex in the previous
 *                 superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<LongWritable, ARPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    if (getSuperstep() == 0) {
        if (!isPartitioned) {
            setVertexStartValue(vertex);
        }
        String aggregator = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get();
        notifyAggregator(aggregator, POSITIVE_ONE);
        sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition());
    } else {
        // even superstep: migrate phase
        if ((getSuperstep() % 2) == 0) {
            long desiredPartition = vertex.getValue().getDesiredPartition().get();
            long currentPartition = vertex.getValue().getCurrentPartition().get();
            long stableCounter = vertex.getValue().getStableCounter().get();
            if (desiredPartition != currentPartition) {
                boolean migrate = isAllowedToMigrate(desiredPartition);
                if (migrate) {
                    migrateVertex(vertex, desiredPartition);
                    sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition());
                    vertex.getValue().setStableCounter(new LongWritable(0));
                } else {
                    stableCounter++;
                    vertex.getValue().setStableCounter(new LongWritable(stableCounter));
                }
            }
            vertex.voteToHalt();
        } else { // odd superstep: demand phase
            if (vertex.getValue().getStableCounter().get() < stableThreshold) {
                long desiredPartition = getDesiredPartition(vertex, messages);
                vertex.getValue().setDesiredPartition(new LongWritable(desiredPartition));
                long currentValue = vertex.getValue().getCurrentPartition().get();
                boolean changed = currentValue != desiredPartition;
                if (changed) {
                    notifyAggregator(DEMAND_AGGREGATOR_PREFIX + desiredPartition, POSITIVE_ONE);
                }
            } else {
                vertex.voteToHalt();
            }
        }
    }
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get the current partition/*from  w ww .j  ava 2  s.co m*/
 *
 * @return the current partition
 */
public LongWritable getCurrentPartition() {
    return new LongWritable(this.currentPartition);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get method to get the desired partition
 *
 * @return the desired Partition//from ww  w. j  a  v a2 s . c o  m
 */
public LongWritable getDesiredPartition() {
    return new LongWritable(this.desiredPartition);
}

From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java

License:Open Source License

/**
 * Get the current stable counter/*  w w  w .  j av a  2  s. c  o  m*/
 *
 * @return the stable counter
 */
public LongWritable getStableCounter() {
    return new LongWritable(this.stableCounter);
}

From source file:de.unileipzig.dbs.giraph.algorithms.labelpropagation.LPComputation.java

License:Open Source License

/**
 * The actual LabelPropagation Computation
 *
 * @param vertex   Vertex/*from   ww w. j a  v  a  2  s .  c  om*/
 * @param messages Messages that were sent to this vertex in the previous
 *                 superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<LongWritable, LPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    if (getSuperstep() == 0) {
        sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity());
    } else {
        long currentCommunity = vertex.getValue().getCurrentCommunity().get();
        long lastCommunity = vertex.getValue().getLastCommunity().get();
        long newCommunity = getNewCommunity(vertex, messages);
        long currentStabilizationRound = vertex.getValue().getStabilizationRounds();

        // increment the stabilization count if vertex wants to stay in the
        // same partition
        if (lastCommunity == newCommunity) {
            currentStabilizationRound++;
            vertex.getValue().setStabilizationRounds(currentStabilizationRound);
        }

        boolean isUnstable = currentStabilizationRound <= stabilizationRounds;
        boolean mayChange = currentCommunity != newCommunity;
        if (mayChange && isUnstable) {
            vertex.getValue().setLastCommunity(new LongWritable(currentCommunity));
            vertex.getValue().setCurrentCommunity(new LongWritable(newCommunity));
            // reset stabilization counter
            vertex.getValue().setStabilizationRounds(0);
            sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity());
        }
    }
    vertex.voteToHalt();
}