List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.deduplication.DocumentInfo.java
License:Apache License
public void setDocSimHash(LongWritable docSimHash) { this.docSimHash = new LongWritable(docSimHash.get()); }
From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.statistics.helper.TextLongCountingReducer.java
License:Apache License
@Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable intWritable : values) { sum += intWritable.get();/* w ww. j av a2s . c om*/ } context.write(key, new LongWritable(sum)); }
From source file:de.tudarmstadt.ukp.dkpro.c4corpus.hadoop.utils.URIExtractorTest.java
License:Apache License
@Test public void testMapper() throws IOException, InterruptedException { final String expectedURI = "https://www.ukp.tu-darmstadt.de/ukp-home/"; final WARCWritable warc = EasyMock.mock(WARCWritable.class); final WARCRecord record = EasyMock.mock(WARCRecord.class); final WARCRecord.Header header = EasyMock.mock(WARCRecord.Header.class); @SuppressWarnings("unchecked") final URIExtractor.URIExtractorMapper.Context context = EasyMock .mock(URIExtractor.URIExtractorMapper.Context.class); EasyMock.expect(record.getHeader()).andReturn(header); EasyMock.expect(warc.getRecord()).andReturn(record); EasyMock.expect(header.getTargetURI()).andReturn(expectedURI); context.write(new Text(expectedURI), NullWritable.get()); EasyMock.replay(warc, record, header, context); final URIExtractor.URIExtractorMapper mapper = new URIExtractor.URIExtractorMapper(); mapper.map(new LongWritable(0), warc, context); EasyMock.verify(warc, record, header, context); }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java
License:Open Source License
/** * Moves a vertex from its old to its new partition. * * @param vertex vertex//from w ww . j a va 2 s . co m * @param desiredPartition partition to move vertex to */ private void migrateVertex(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex, long desiredPartition) { // add current partition to partition history vertex.getValue().addToPartitionHistory(vertex.getValue().getCurrentPartition().get()); // decrease capacity in old partition String oldPartition = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get(); notifyAggregator(oldPartition, NEGATIVE_ONE); // increase capacity in new partition String newPartition = CAPACITY_AGGREGATOR_PREFIX + desiredPartition; notifyAggregator(newPartition, POSITIVE_ONE); vertex.getValue().setCurrentPartition(new LongWritable(desiredPartition)); }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java
License:Open Source License
/** * Initializes the vertex with a partition id. This is calculated using * modulo (vertex-id % number of partitions). * * @param vertex vertex//from w ww .jav a 2s . c o m */ private void setVertexStartValue(final Vertex<LongWritable, ARPVertexValue, NullWritable> vertex) { long startValue = vertex.getId().get() % k; vertex.getValue().setCurrentPartition(new LongWritable(startValue)); }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPComputation.java
License:Open Source License
/** * The actual ADP computation./*from ww w . jav a2 s .c om*/ * * @param vertex Vertex * @param messages Messages that were sent to this vertex in the previous * superstep. * @throws IOException */ @Override public void compute(Vertex<LongWritable, ARPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages) throws IOException { if (getSuperstep() == 0) { if (!isPartitioned) { setVertexStartValue(vertex); } String aggregator = CAPACITY_AGGREGATOR_PREFIX + vertex.getValue().getCurrentPartition().get(); notifyAggregator(aggregator, POSITIVE_ONE); sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition()); } else { // even superstep: migrate phase if ((getSuperstep() % 2) == 0) { long desiredPartition = vertex.getValue().getDesiredPartition().get(); long currentPartition = vertex.getValue().getCurrentPartition().get(); long stableCounter = vertex.getValue().getStableCounter().get(); if (desiredPartition != currentPartition) { boolean migrate = isAllowedToMigrate(desiredPartition); if (migrate) { migrateVertex(vertex, desiredPartition); sendMessageToAllEdges(vertex, vertex.getValue().getCurrentPartition()); vertex.getValue().setStableCounter(new LongWritable(0)); } else { stableCounter++; vertex.getValue().setStableCounter(new LongWritable(stableCounter)); } } vertex.voteToHalt(); } else { // odd superstep: demand phase if (vertex.getValue().getStableCounter().get() < stableThreshold) { long desiredPartition = getDesiredPartition(vertex, messages); vertex.getValue().setDesiredPartition(new LongWritable(desiredPartition)); long currentValue = vertex.getValue().getCurrentPartition().get(); boolean changed = currentValue != desiredPartition; if (changed) { notifyAggregator(DEMAND_AGGREGATOR_PREFIX + desiredPartition, POSITIVE_ONE); } } else { vertex.voteToHalt(); } } } }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java
License:Open Source License
/** * Get the current partition/*from w ww .j ava 2 s.co m*/ * * @return the current partition */ public LongWritable getCurrentPartition() { return new LongWritable(this.currentPartition); }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java
License:Open Source License
/** * Get method to get the desired partition * * @return the desired Partition//from ww w. j a v a2 s . c o m */ public LongWritable getDesiredPartition() { return new LongWritable(this.desiredPartition); }
From source file:de.unileipzig.dbs.giraph.algorithms.adaptiverepartitioning.ARPVertexValue.java
License:Open Source License
/** * Get the current stable counter/* w w w . j av a 2 s. c o m*/ * * @return the stable counter */ public LongWritable getStableCounter() { return new LongWritable(this.stableCounter); }
From source file:de.unileipzig.dbs.giraph.algorithms.labelpropagation.LPComputation.java
License:Open Source License
/** * The actual LabelPropagation Computation * * @param vertex Vertex/*from ww w. j a v a 2 s . c om*/ * @param messages Messages that were sent to this vertex in the previous * superstep. * @throws IOException */ @Override public void compute(Vertex<LongWritable, LPVertexValue, NullWritable> vertex, Iterable<LongWritable> messages) throws IOException { if (getSuperstep() == 0) { sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity()); } else { long currentCommunity = vertex.getValue().getCurrentCommunity().get(); long lastCommunity = vertex.getValue().getLastCommunity().get(); long newCommunity = getNewCommunity(vertex, messages); long currentStabilizationRound = vertex.getValue().getStabilizationRounds(); // increment the stabilization count if vertex wants to stay in the // same partition if (lastCommunity == newCommunity) { currentStabilizationRound++; vertex.getValue().setStabilizationRounds(currentStabilizationRound); } boolean isUnstable = currentStabilizationRound <= stabilizationRounds; boolean mayChange = currentCommunity != newCommunity; if (mayChange && isUnstable) { vertex.getValue().setLastCommunity(new LongWritable(currentCommunity)); vertex.getValue().setCurrentCommunity(new LongWritable(newCommunity)); // reset stabilization counter vertex.getValue().setStabilizationRounds(0); sendMessageToAllEdges(vertex, vertex.getValue().getCurrentCommunity()); } } vertex.voteToHalt(); }