Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get() 

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:at.illecker.hama.rootbeer.examples.piestimator.cpu.PiEstimatorCpuBSP.java

License:Apache License

@Override
public void cleanup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException {

    // MasterTask writes out results
    if (peer.getPeerName().equals(m_masterTask)) {

        int numMessages = peer.getNumCurrentMessages();

        long totalHits = 0;
        LongWritable received;
        while ((received = peer.getCurrentMessage()) != null) {
            totalHits += received.get();
        }/*  w w  w. j av a2 s .  c o  m*/

        double pi = 4.0 * totalHits / (m_calculationsPerBspTask * numMessages);

        // DEBUG
        if (m_isDebuggingEnabled) {
            // Write log to dfs
            BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
            FileSystem fs = FileSystem.get(peer.getConfiguration());
            FSDataOutputStream outStream = fs
                    .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

            outStream.writeChars("BSP=PiEstimatorCpuBSP,Iterations=" + m_iterations + "\n");

            outStream.writeChars("totalHits: " + totalHits + "\n");
            outStream.writeChars("numMessages: " + numMessages + "\n");
            outStream.writeChars("calculationsPerBspTask: " + m_calculationsPerBspTask + "\n");
            outStream.writeChars("calculationsTotal: " + (m_calculationsPerBspTask * numMessages) + "\n");
            outStream.close();
        }

        peer.write(
                new Text("Estimated value of PI(3,14159265) using " + (m_calculationsPerBspTask * numMessages)
                // + (peer.getNumPeers() * m_threadCount * m_iterations)
                        + " points is"),
                new DoubleWritable(pi));
    }
}

---

From source file:at.illecker.hama.rootbeer.examples.piestimator.gpu.PiEstimatorGpuBSP.java

License:Apache License

@Override
public void cleanup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException {

    if (peer.getPeerName().equals(m_masterTask)) {

        long totalHits = 0;
        LongWritable received;
        while ((received = peer.getCurrentMessage()) != null) {
            totalHits += received.get();
        }/* www  . j a va 2 s  .  c  om*/

        double pi = 4.0 * totalHits / (m_calculationsPerThread * m_blockSize * m_gridSize);

        peer.write(
                new Text("Estimated value of PI(3,14159265) using "
                        + (m_calculationsPerThread * m_blockSize * m_gridSize) + " points is"),
                new DoubleWritable(pi));
    }
}

---

From source file:backup.store.ExternalExtendedBlockSort.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path dir = new Path("file:///home/apm/Development/git-projects/hdfs-backup/hdfs-backup-core/tmp");
    dir.getFileSystem(conf).delete(dir, true);
    long start = System.nanoTime();
    try (ExternalExtendedBlockSort<LongWritable> sort = new ExternalExtendedBlockSort<>(conf, dir,
            LongWritable.class)) {
        Random random = new Random();
        for (int bp = 0; bp < 1; bp++) {
            String bpid = UUID.randomUUID().toString();
            for (int i = 0; i < 10000000; i++) {
                // for (int i = 0; i < 10; i++) {
                long genstamp = random.nextInt(20000);
                long blockId = random.nextLong();
                ExtendedBlock extendedBlock = new ExtendedBlock(bpid, blockId,
                        random.nextInt(Integer.MAX_VALUE), genstamp);
                sort.add(extendedBlock, new LongWritable(blockId));
            }//from  www . j a  v a 2  s .  co  m
        }
        System.out.println("finished");
        sort.finished();
        System.out.println("interate");
        for (String blockPoolId : sort.getBlockPoolIds()) {
            ExtendedBlockEnum<LongWritable> blockEnum = sort.getBlockEnum(blockPoolId);
            ExtendedBlock block;
            long l = 0;
            while ((block = blockEnum.next()) != null) {
                // System.out.println(block);
                long blockId = block.getBlockId();
                l += blockId;
                LongWritable currentValue = blockEnum.currentValue();
                if (currentValue.get() != blockId) {
                    System.err.println("Error " + blockId);
                }
            }
            System.out.println(l);
        }
    }
    long end = System.nanoTime();
    System.out.println("Time [" + (end - start) / 1000000.0 + " ms]");
}

---

From source file:bbuzz2011.stackoverflow.preprocess.xml.StackOverflowPostXMLMapper.java

License:Apache License

private void writePostBody(LongWritable key, Text value, Context context)
        throws SAXException, IOException, XPathExpressionException, InterruptedException {
    // TODO Where counters used? May be for some statistics?
    // Are counters global and atomic for all mappers?
    // Where do them output?
    context.getCounter(StackOverflowPostXMLMapper.Counter.TITLES).increment(1);

    Document doc = documentBuilder.parse(new InputSource(new StringReader(value.toString())));

    // Retrieve title from xml post using xpath
    String title = (String) postTitleXPath.evaluate(doc, XPathConstants.STRING);
    if (title == null || title.equals("")) {
        context.getCounter(Counter.MISSING_TITLES).increment(1);
        return;/* ww w  . j a  va 2s .  c o  m*/
    }

    String postHtml = (String) postBodyXPath.evaluate(doc, XPathConstants.STRING);
    String content = parser.parsePostContent(postHtml);

    // TODO Why not stackexchange post Id attribute?
    postKey.set((int) key.get());

    postWritable.setTitle(title);
    postWritable.setContent(content);

    // Retrieve questions, not answers
    // TODO as improvement we can combine question and answers as single document for better clustering.
    if (isQuestion(doc)) {
        context.getCounter(Counter.QUESTIONS).increment(1);
        context.write(postKey, postWritable);
    }
}

---

From source file:be.uantwerpen.adrem.eclat.EclatMinerReducerSetCount.java

License:Apache License

@Override
public void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    long levelTotal = 0;
    for (LongWritable lw : values) {
        levelTotal += lw.get();
    }/*from w  ww . ja va 2s.  c o  m*/
    total += levelTotal;
    context.write(key, new LongWritable(levelTotal));
}

---

From source file:boostingPL.MR.AdaBoostPLTestMapper.java

License:Open Source License

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    Instance inst = InstancesHelper.createInstance(value.toString(), insts);
    try {//from   w  w  w. j a v  a  2 s .  co m
        eval.evaluateModelOnceAndRecordPrediction(boostingPL, inst);
    } catch (Exception e) {
        LOG.warn("[BoostingPL-Test]: Evalute instance error!, key = " + key.get());
        e.printStackTrace();
    }
    instanceCounter.increment(1);
    context.write(key, value);
}

---

From source file:boostingPL.MR.AdaBoostPLTestReducer.java

License:Open Source License

protected void reduce(LongWritable key, Iterable<Text> value, Context context)
        throws IOException, InterruptedException {
    for (Text t : value) {
        Instance inst = InstancesHelper.createInstance(t.toString(), insts);
        try {/*from   ww  w . jav a 2s . c o m*/
            eval.evaluateModelOnceAndRecordPrediction(boostingPL, inst);
        } catch (Exception e) {
            LOG.warn("[BoostingPL-Test]: Evalute instance error!, key = " + key.get());
            e.printStackTrace();
        }
    }
}

---

From source file:ca.nlap.giraphsociallayout.ConnectedComponentsComputation.java

License:Apache License

/**
 * Propagates the smallest vertex id to all neighbors. Will always choose to
 * halt and only reactivate if a smaller id has been sent to it.
 *
 * @param vertex Vertex// w  w  w . j  av a  2s  .c o  m
 * @param messages Iterator of messages from the previous superstep.
 * @throws IOException
 */
@Override
public void compute(Vertex<LongWritable, LongWritable, NullWritable> vertex, Iterable<LongWritable> messages)
        throws IOException {
    long currentComponent = vertex.getValue().get();

    // First superstep is special, because we can simply look at the neighbors
    if (getSuperstep() == 0) {
        for (Edge<LongWritable, NullWritable> edge : vertex.getEdges()) {
            long neighbor = edge.getTargetVertexId().get();
            if (neighbor < currentComponent) {
                currentComponent = neighbor;
            }
        }
        // Only need to send value if it is not the own id
        if (currentComponent != vertex.getValue().get()) {
            vertex.setValue(new LongWritable(currentComponent));
            for (Edge<LongWritable, NullWritable> edge : vertex.getEdges()) {
                LongWritable neighbor = edge.getTargetVertexId();
                if (neighbor.get() > currentComponent) {
                    sendMessage(neighbor, vertex.getValue());
                }
            }
        }

        vertex.voteToHalt();
        return;
    }

    boolean changed = false;
    // did we get a smaller id ?
    for (LongWritable message : messages) {
        long candidateComponent = message.get();
        if (candidateComponent < currentComponent) {
            currentComponent = candidateComponent;
            changed = true;
        }
    }

    // propagate new component id to the neighbors
    if (changed) {
        vertex.setValue(new LongWritable(currentComponent));
        sendMessageToAllEdges(vertex, vertex.getValue());
    }
    vertex.voteToHalt();
}

---

From source file:ca.sparkera.adapters.mapred.MainframeVBRecordReader.java

License:Apache License

@Override
public synchronized boolean next(LongWritable key, BytesWritable value) throws IOException {
    boolean dataRead = reader.nextKeyValue();
    if (dataRead) {
        LongWritable newKey = reader.getCurrentKey();
        BytesWritable newValue = reader.getCurrentValue();
        key.set(newKey.get());
        value.set(newValue);// w w  w  .java 2s  .  c om
    }
    return dataRead;
}

---

From source file:ca.uwaterloo.iss4e.hive.UDAFPercentile.java

License:Apache License

/**
 * Increment the State object with o as the key, and i as the count.
 *//* w ww  . j a  v  a  2s.  c  om*/
private static void increment(State s, LongWritable o, long i) {
    if (s.counts == null) {
        s.counts = new HashMap<LongWritable, LongWritable>();
    }
    LongWritable count = s.counts.get(o);
    if (count == null) {
        // We have to create a new object, because the object o belongs
        // to the code that creates it and may get its value changed.
        LongWritable key = new LongWritable();
        key.set(o.get());
        s.counts.put(key, new LongWritable(i));
    } else {
        count.set(count.get() + i);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »