Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get()

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:org.apache.blur.manager.writer.SnapshotIndexDeletionPolicy.java

License:Apache License

private void loadGenerations() throws IOException {
    FileSystem fileSystem = _path.getFileSystem(_configuration);
    FileStatus[] listStatus = fileSystem.listStatus(_path);
    SortedSet<FileStatus> existing = new TreeSet<FileStatus>(Arrays.asList(listStatus));
    if (existing.isEmpty()) {
        return;//from   ww w  . j  av  a 2 s.c  o  m
    }
    FileStatus last = existing.last();
    Reader reader = new SequenceFile.Reader(fileSystem, last.getPath(), _configuration);
    Text key = new Text();
    LongWritable value = new LongWritable();
    while (reader.next(key, value)) {
        String name = key.toString();
        long gen = value.get();
        _namesToGenerations.put(name, gen);
        Set<String> names = _generationsToNames.get(gen);
        if (names == null) {
            names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
            _generationsToNames.put(gen, names);
        }
        names.add(name);
    }
    reader.close();
    existing.remove(last);
    cleanupOldFiles(fileSystem, existing);
}

From source file:org.apache.flink.formats.sequencefile.SequenceStreamingFileSinkITCase.java

License:Apache License

private List<Tuple2<Long, String>> readSequenceFile(File file) throws IOException {
    SequenceFile.Reader reader = new SequenceFile.Reader(configuration,
            SequenceFile.Reader.file(new org.apache.hadoop.fs.Path(file.toURI())));
    LongWritable key = new LongWritable();
    Text val = new Text();
    ArrayList<Tuple2<Long, String>> results = new ArrayList<>();
    while (reader.next(key, val)) {
        results.add(new Tuple2<>(key.get(), val.toString()));
    }// ww w  .  j av a2 s. co  m
    reader.close();
    return results;
}

From source file:org.apache.giraph.aggregators.LongMaxAggregator.java

License:Apache License

@Override
public void aggregate(LongWritable value) {
    getAggregatedValue().set(Math.max(getAggregatedValue().get(), value.get()));
}

From source file:org.apache.giraph.aggregators.LongMinAggregator.java

License:Apache License

@Override
public void aggregate(LongWritable value) {
    getAggregatedValue().set(Math.min(getAggregatedValue().get(), value.get()));
}

From source file:org.apache.giraph.aggregators.LongOverwriteAggregator.java

License:Apache License

@Override
public void aggregate(LongWritable value) {
    getAggregatedValue().set(value.get());
}

From source file:org.apache.giraph.aggregators.LongProductAggregator.java

License:Apache License

@Override
public void aggregate(LongWritable value) {
    getAggregatedValue().set(getAggregatedValue().get() * value.get());
}

From source file:org.apache.giraph.aggregators.LongSumAggregator.java

License:Apache License

@Override
public void aggregate(LongWritable value) {
    getAggregatedValue().set(getAggregatedValue().get() + value.get());
}

From source file:org.apache.giraph.block_app.framework.BlockExecutionTest.java

License:Apache License

@Test
public void testReducing() {
    TestGraph<LongWritable, LongWritable, NullWritable> graph = createTestGraph();

    final LongWritable value = new LongWritable();

    LocalBlockRunner.runBlock(graph, new Piece<WritableComparable, Writable, Writable, NoMessage, Object>() {
        private ReducerHandle<LongWritable, LongWritable> numVertices;

        @Override/*w ww. j  a  va 2s.  c  o m*/
        public void registerReducers(CreateReducersApi reduceApi, Object executionStage) {
            numVertices = reduceApi.createLocalReducer(SumReduce.LONG);
        }

        @Override
        public VertexSender<WritableComparable, Writable, Writable> getVertexSender(
                BlockWorkerSendApi<WritableComparable, Writable, Writable, NoMessage> workerApi,
                Object executionStage) {

            return new InnerVertexSender() {
                @Override
                public void vertexSend(Vertex<WritableComparable, Writable, Writable> vertex) {
                    numVertices.reduce(new LongWritable(1));
                }
            };
        }

        @Override
        public void masterCompute(BlockMasterApi masterApi, Object executionStage) {
            value.set(numVertices.getReducedValue(masterApi).get());
        }
    }, new Object());

    Assert.assertEquals(4, value.get());
}

From source file:org.apache.giraph.block_app.library.prepare_graph.WeaklyConnectedComponents.java

License:Apache License

/**
 * This is just used internally in weakly connected components if the graph
 * has float weights. This works fine here because we don't actually ever
 * use the weights, but should not be used outside of the
 * WeaklyConnectedComponents class and hence is private.
 *///from w w  w  .  j a v a2 s .com
private static <V extends Writable> Piece<LongWritable, V, FloatWritable, LongWritable, Object> makeSymmetricFloatWeighted() {
    LongSet set = new LongOpenHashSet();
    FloatWritable floatWritable = new FloatWritable(1.0f);
    ConsumerWithVertex<LongWritable, V, FloatWritable, Iterable<LongWritable>> addEdges = (vertex,
            neighbors) -> {
        set.clear();
        for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) {
            set.add(edge.getTargetVertexId().get());
        }
        for (LongWritable message : neighbors) {
            if (!set.contains(message.get())) {
                Edge<LongWritable, FloatWritable> edge = EdgeFactory.create(new LongWritable(message.get()),
                        floatWritable);
                vertex.addEdge(edge);
                set.add(message.get());
            }
        }
    };
    return Pieces.sendMessageToNeighbors("MakeSymmetricFloatWeighted", LongWritable.class,
            VertexSuppliers.<LongWritable, V, FloatWritable>vertexIdSupplier(), addEdges);
}

From source file:org.apache.giraph.block_app.library.stats.PartitioningStats.java

License:Apache License

/**
 * Calculate edge locality - ratio of edges that are within a same bucket.
 */// www  . j a v  a2 s  .c  om
public static <V extends Writable> Block calculateEdgeLocality(
        SupplierFromVertex<WritableComparable, V, Writable, LongWritable> bucketSupplier,
        DoubleConsumer edgeLocalityConsumer) {
    final Pair<LongWritable, LongWritable> pair = Pair.of(new LongWritable(), new LongWritable());
    return SendMessageChain
            .<WritableComparable, V, Writable, LongWritable>startSendToNeighbors("CalcLocalEdgesPiece",
                    LongWritable.class, bucketSupplier)
            .endReduceWithMaster("AggregateEdgeLocalityPiece", new PairReduce<>(SumReduce.LONG, SumReduce.LONG),
                    (vertex, messages) -> {
                        long bucket = bucketSupplier.get(vertex).get();
                        int local = 0;
                        int total = 0;
                        for (LongWritable otherCluster : messages) {
                            total++;
                            if (bucket == otherCluster.get()) {
                                local++;
                            }
                        }
                        pair.getLeft().set(local);
                        pair.getRight().set(total);
                        return pair;
                    }, (reducedPair, master) -> {
                        long localEdges = reducedPair.getLeft().get();
                        long totalEdges = reducedPair.getRight().get();
                        double edgeLocality = (double) localEdges / totalEdges;
                        LOG.info("locality ratio = " + edgeLocality);
                        master.getCounter("Edge locality stats", "edge locality (in percent * 1000)")
                                .setValue((long) (edgeLocality * 100000));
                        edgeLocalityConsumer.apply(edgeLocality);
                    });
}