Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get()

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:org.apache.giraph.block_app.library.stats.PartitioningStats.java

License:Apache License

/**
 * Calculates average fanout - average number of distinct buckets that vertex
 * has neighbors in.// w w  w.  ja va 2 s. c  o  m
 */
public static <V extends Writable> Block calculateFanout(
        SupplierFromVertex<WritableComparable, V, Writable, LongWritable> bucketSupplier,
        DoubleConsumer averageFanoutConsumer) {
    final Pair<LongWritable, LongWritable> pair = Pair.of(new LongWritable(), new LongWritable(1));
    return SendMessageChain.<WritableComparable, V, Writable, LongWritable>startSendToNeighbors(
            "CalcFanoutPiece", LongWritable.class, bucketSupplier).endReduceWithMaster("AggregateFanoutPiece",
                    new PairReduce<>(SumReduce.LONG, SumReduce.LONG), (vertex, messages) -> {
                        LongSet setOfNeighborBuckets = new LongOpenHashSet();
                        for (LongWritable neighborBucket : messages) {
                            setOfNeighborBuckets.add(neighborBucket.get());
                        }
                        pair.getLeft().set(setOfNeighborBuckets.size());
                        return pair;
                    }, (reducedPair, master) -> {
                        long fanout = reducedPair.getLeft().get();
                        long numVertices = reducedPair.getRight().get();
                        double avgFanout = (double) fanout / numVertices;
                        LOG.info("fanout ratio = " + avgFanout);
                        master.getCounter("Fanout stats", "fanout (in percent * 1000)")
                                .setValue((long) (avgFanout * 100000));
                        averageFanoutConsumer.apply(avgFanout);
                    });
}

From source file:org.apache.giraph.block_app.library.striping.StripingUtils.java

License:Apache License

/**
 * Fast hash-based striping for LongWritable IDs, returns a function
 * that for a given ID returns it's stripe index.
 *///from w  ww  .j  a  v a 2  s. co  m
public static Obj2IntFunction<LongWritable> fastHashStriping(final int stripes) {
    return new Obj2IntFunction<LongWritable>() {
        @Override
        public int apply(LongWritable id) {
            return fastStripe(id.get(), stripes);
        }
    };
}

From source file:org.apache.giraph.block_app.library.striping.StripingUtils.java

License:Apache License

/**
 * Fast hash-based striping for LongWritable IDs, returns a function
 * that for a given stripe index returns a predicate checking whether ID is
 * in that stripe.//from   www  .jav a 2 s  . c  om
 */
public static Int2ObjFunction<Predicate<LongWritable>> fastHashStripingPredicate(final int stripes) {
    return new Int2ObjFunction<Predicate<LongWritable>>() {
        @Override
        public Predicate<LongWritable> apply(final int stripe) {
            return new Predicate<LongWritable>() {
                @Override
                public boolean apply(LongWritable id) {
                    return fastStripe(id.get(), stripes) == stripe;
                }
            };
        }
    };
}

From source file:org.apache.giraph.block_app.library.TestMessageChain.java

License:Apache License

private static long max(Iterable<LongWritable> messages) {
    long result = 0;
    for (LongWritable message : messages) {
        result = Math.max(result, message.get());
    }//from  w w w.  jav a  2  s  . c om
    return result;
}

From source file:org.apache.giraph.combiner.MinimumLongMessageCombiner.java

License:Apache License

@Override
public void combine(LongWritable vertexIndex, LongWritable originalMessage, LongWritable messageToCombine) {
    if (originalMessage.get() > messageToCombine.get()) {
        originalMessage.set(messageToCombine.get());
    }//from   w w w . j av a  2s . c om
}

From source file:org.apache.giraph.comm.messages.primitives.LongByteArrayMessageStore.java

License:Apache License

@Override
public void addPartitionMessage(int partitionId, LongWritable destVertexId, M message) throws IOException {
    Long2ObjectOpenHashMap<DataInputOutput> partitionMap = map.get(partitionId);

    synchronized (partitionMap) {
        DataInputOutput dataInputOutput = getDataInputOutput(partitionMap, destVertexId.get());
        message.write(dataInputOutput.getDataOutput());
    }/*from  w w w .  j a v a 2s . c o m*/
}

From source file:org.apache.giraph.comm.messages.primitives.LongByteArrayMessageStore.java

License:Apache License

@Override
public boolean hasMessagesForVertex(LongWritable vertexId) {
    Long2ObjectOpenHashMap<?> partitionMap = getPartitionMap(vertexId);

    if (partitionMap == null) {
        return false;
    }/*from w  w w. j  a  v a  2 s.co m*/

    synchronized (partitionMap) {
        return partitionMap.containsKey(vertexId.get());
    }
}

From source file:org.apache.giraph.comm.messages.primitives.LongByteArrayMessageStore.java

License:Apache License

@Override
public Iterable<M> getVertexMessages(LongWritable vertexId) throws IOException {
    Long2ObjectOpenHashMap<DataInputOutput> partitionMap = getPartitionMap(vertexId);

    if (partitionMap == null) {
        return EmptyIterable.get();
    }/* w  w w  .  jav a  2 s  .  co  m*/

    // YH: must synchronize, as writes are concurrent w/ reads in async
    synchronized (partitionMap) {
        DataInputOutput dataInputOutput = partitionMap.get(vertexId.get());
        if (dataInputOutput == null) {
            return EmptyIterable.get();
        } else {
            return new MessagesIterable<M>(dataInputOutput, messageValueFactory);
        }
    }
}

From source file:org.apache.giraph.comm.messages.primitives.LongByteArrayMessageStore.java

License:Apache License

@Override
public Iterable<M> removeVertexMessages(LongWritable vertexId) throws IOException {
    Long2ObjectOpenHashMap<DataInputOutput> partitionMap = getPartitionMap(vertexId);

    if (partitionMap == null) {
        return EmptyIterable.get();
    }/*from  www . j a  va2s.com*/

    // YH: must synchronize, as writes are concurrent w/ reads in async
    synchronized (partitionMap) {
        DataInputOutput dataInputOutput = partitionMap.remove(vertexId.get());
        if (dataInputOutput == null) {
            return EmptyIterable.get();
        } else {
            return new MessagesIterable<M>(dataInputOutput, messageValueFactory);
        }
    }
}

From source file:org.apache.giraph.comm.messages.primitives.LongByteArrayMessageStore.java

License:Apache License

@Override
public void clearVertexMessages(LongWritable vertexId) throws IOException {
    // YH: not used in async, but synchronize anyway
    Long2ObjectOpenHashMap<DataInputOutput> partitionMap = getPartitionMap(vertexId);

    if (partitionMap == null) {
        return;//from  w  w  w  .j  ava 2s  . c o  m
    }

    synchronized (partitionMap) {
        partitionMap.remove(vertexId.get());
    }
}