Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get()

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:org.sleuthkit.hadoop.GrepCountReducer.java

License:Open Source License

@SuppressWarnings("unchecked")
@Override/*from   w w  w  .  j  av  a 2s  . c o m*/
protected void reduce(LongWritable key, Iterable<LongWritable> values, Context context) {
    JSONObject obj = new JSONObject();
    int sum = 0;
    for (LongWritable value : values) {
        sum += value.get();
    }
    obj.put("a", key.get());
    obj.put("n", sum);
    obj.put("kw", regexes[(int) key.get()]);

    outArray.add(obj);
}

From source file:org.sleuthkit.hadoop.match.GrepCountReducer.java

License:Open Source License

@SuppressWarnings("unchecked")
@Override//w w  w. j  av  a2s  .c om
protected void reduce(LongWritable key, Iterable<LongWritable> values, Context context) throws IOException {
    gen.writeStartObject();

    int sum = 0;
    for (LongWritable value : values) {
        sum += value.get();
    }

    gen.writeNumberField("a", key.get());
    gen.writeNumberField("n", sum);
    gen.writeStringField("kw", regexes[(int) key.get()]);

    gen.writeEndObject();
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CountPagesReducer.java

License:Open Source License

@Override
public void reduce(Text key, Iterator<LongWritable> values, OutputCollector<Text, LongWritable> output,
        Reporter report) throws IOException {
    long sum = 0;
    while (values.hasNext()) {
        LongWritable value = values.next();
        sum += value.get();
    }//from  ww  w . j  a va 2s  .  c  o  m
    output.collect(key, new LongWritable(sum));
}

From source file:org.smartfrog.services.hadoop.mapreduce.terasort.TeraGenMapper.java

License:Open Source License

public void map(LongWritable row, NullWritable ignored, OutputCollector<Text, Text> output, Reporter reporter)
        throws IOException {
    long rowId = row.get();
    if (rand == null) {
        // we use 3 random numbers per a row
        rand = new TeraGenRandomGenerator(rowId * 3);
    }/* w  ww. ja v  a  2 s .  c  o  m*/
    addKey();
    value.clear();
    addRowId(rowId);
    addFiller(rowId);
    output.collect(key, value);
}

From source file:org.unigram.likelike.common.RelatedUsersWritable.java

License:Apache License

/**
 * write./*from   w ww .  j  a v  a  2  s . com*/
 * @param out output stream
 * @throws IOException -
 */
@Override
public void write(final DataOutput out) throws IOException {
    out.writeInt(this.relatedUsers.size());
    for (LongWritable item : this.relatedUsers) {
        out.writeLong(item.get());
    }
}

From source file:org.unigram.likelike.lsh.GetRecommendationsReducer.java

License:Apache License

/**
 * reduce. //  w w w.j a  va  2  s  .  c  om
 * @param key target
 * @param values candidates
 * @param context -
 * @throws IOException - 
 * @throws InterruptedException -
 */
public void reduce(final LongWritable key, final Iterable<Candidate> values, final Context context)
        throws IOException, InterruptedException {

    HashMap<Long, Double> candidates = new HashMap<Long, Double>();
    for (Candidate cand : values) {
        Long tid = cand.getId().get();
        if (candidates.containsKey(tid)) {
            Double weight = candidates.get(tid);
            weight += 1.0; // not use the size of the cluster
            candidates.put(tid, weight);
        } else {
            candidates.put(tid, new Double(1.0));
        }

        if (candidates.size() > 50000) { // TODO should be parameterized
            break;
        }
    }

    /* sort by value and then output */
    ArrayList<Map.Entry> array = new ArrayList<Map.Entry>(candidates.entrySet());
    Collections.sort(array, this.comparator);

    Iterator it = array.iterator();
    int i = 0;
    while (it.hasNext()) {
        if (i >= this.maxOutputSize) {
            return;
        }
        Map.Entry obj = (Map.Entry) it.next();
        try {
            this.writer.write(key.get(), (Long) obj.getKey(), context);
        } catch (Exception e) {
            e.printStackTrace();
        }
        i += 1;
    }
}

From source file:org.unigram.likelike.lsh.SelectClustersMapper.java

License:Apache License

/**
 * map./*w ww . j av  a  2 s.  c  o  m*/
 * @param key dummy
 * @param value containing id and the features
 * @param context context 
 * @exception IOException -
 * @exception InterruptedException -
 */
@Override
public final void map(final LongWritable key, final Text value, final Context context)
        throws IOException, InterruptedException {
    String inputStr = value.toString();

    try {
        String[] tokens = inputStr.split("\t");
        Long id = Long.parseLong(tokens[0]); // example id
        Set<Long> featureSet = this.extractFeatures(tokens[1]);

        for (int i = 0; i < seedsAry.length; i++) {
            LongWritable clusterId = this.function.returnClusterId(featureSet, seedsAry[i]);
            context.write(new SeedClusterId(seedsAry[i], clusterId.get()), new RelatedUsersWritable(id));
        }
    } catch (ArrayIndexOutOfBoundsException e) {
        System.out.println("PARSING ERROR in line: " + inputStr);
        e.printStackTrace();
    }
}

From source file:org.utils.UnsplittableFileReader.java

License:Apache License

@Override
public synchronized boolean nextKeyValue() throws IOException {
    boolean res = reader.nextKeyValue();
    if (res) {//w  ww .  ja v a2s .c o  m
        LongWritable lineNumber = reader.getCurrentKey();
        Text lineString = reader.getCurrentValue();

        key.clear();
        key.setFilename(filename);
        key.setLine(lineNumber.get());

        value.clear();
        value.set(lineString.copyBytes());

        LOG.debug("read " + key);
    }

    return res;
}

From source file:sa.edu.kaust.twitter.index.TweetsForwardIndex.java

License:Apache License

/**
 * Returns the document vector given a docno.
 *//*from w  w w.j  a v  a2  s. c o m*/
public TweetWritable getValue(long tweetID) throws IOException {
    //System.out.println("tweet-id:  "+tweetID);
    long pos;
    try {
        pos = map.get(tweetID);
    } catch (NoSuchElementException e) {
        return null;
    }
    if (pos == 0)
        return null;
    //System.out.println("pos: "+pos);

    int fileNo = (int) (pos / BuildPostingsForwardIndex.BigNumber);
    pos = pos % BuildPostingsForwardIndex.BigNumber;
    //System.out.println(fileNo+"\t"+pos);
    SequenceFile.Reader reader = new SequenceFile.Reader(mFs,
            new Path(mOrigIndexPath + "/part-" + sFormatW5.format(fileNo)), mConf);

    LongWritable key = new LongWritable();
    TweetWritable value = new TweetWritable();

    /*try {
       value = (ArrayListWritable<IntWritable>) reader.getValueClass().newInstance();
    } catch (Exception e) {
       throw new RuntimeException("Unable to instantiate key/value pair!");
    }*/

    reader.seek(pos);
    reader.next(key, value);

    if (key.get() != tweetID) {
        sLogger.error("unable to find postings for term " + tweetID + ": found term " + key + " instead");
        return null;
    }

    reader.close();
    return value;
}

From source file:smile.wide.algorithms.em.StatEstimator.java

License:Apache License

/**
 * Calls into SMILE to obtain sufficient statistics and log likelihood.
 *///from   w w  w .java  2s .c om
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    if (skipFirstLine && (key.get() == 0)) {
        return; // header line ignored
    }

    String line = value.toString();
    net.clearAllEvidence();
    String[] values = line.split(separator);
    for (int i = 0; i < values.length; i++) {
        if (colMap.containsKey(i)) {
            String v = values[i];
            if (!v.isEmpty() && !v.equals(missingValueToken)) {
                net.setEvidence(colMap.get(i), v);
            }
        }
    }

    double pe = net.probEvidence();

    double[] counts = new double[totalCptSize];
    net.distributedHelperEM(counts);

    DoubleWritable[] peOut = new DoubleWritable[1];
    peOut[0] = new DoubleWritable(Math.log(pe));
    context.write(new IntWritable(-1), new DoubleArrayWritable(peOut));

    int pos = 0;
    for (int h = net.getFirstNode(); h >= 0; h = net.getNextNode(h)) {
        int len = defSizes.get(h);
        DoubleWritable[] countOut = new DoubleWritable[len];
        for (int i = 0; i < len; i++) {
            countOut[i] = new DoubleWritable(counts[pos++]);
        }
        context.write(new IntWritable(h), new DoubleArrayWritable(countOut));
    }
}