Example usage for org.apache.hadoop.io IntWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io IntWritable get.

Prototype

public int get()

Source Link

Document

Return the value of this IntWritable.

Usage

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictReducer.java

License:Open Source License

@Override
public void reduce(IntWritable key, Iterator<Text> iter, OutputCollector<Text, Text> out, Reporter reporter)
        throws IOException {
    OutputCollector<Text, Text> mout = mos.getCollector("vidhashmap" + key.get(), reporter);
    while (iter.hasNext()) {
        Text line = iter.next();/*from   w w w .j  a  v a2  s  . c o  m*/
        mout.collect(null, line);
    }
}

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.TransEdgeReducer.java

License:Open Source License

@Override
public void reduce(IntWritable key, Iterator<Text> iter, OutputCollector<Text, Text> out, Reporter reporter)
        throws IOException {
    if (key.get() != dictionaryId) {
        dictionaryId = key.get();/*  w w w  .  j  a v  a2s.  c om*/
        loadDictionary();
    }

    while (iter.hasNext()) {
        String line = iter.next().toString();
        StringTokenizer tk = new StringTokenizer(line);
        long sourceId = Long.valueOf(tk.nextToken());
        VidType target = vidparser.getValue(tk.nextToken());
        if (dict.containsKey(target)) {
            long targetId = dict.get(target);
            String edata = tk.hasMoreTokens() ? "\t" + tk.nextToken() : "";
            out.collect(null, new Text(sourceId + "\t" + targetId + edata));
        } else {
            LOG.error("Reducer: Cannot find key " + target.toString());
            LOG.error("Line: " + line);
        }
    }
}

From source file:com.intel.hadoop.graphbuilder.partition.mapreduce.vrecord.VrecordIngressReducer.java

License:Open Source License

@Override
public void reduce(IntWritable key, Iterator<Text> value, OutputCollector<Text, Text> out, Reporter reporter)
        throws IOException {
    int numVertices = 0;
    int numOwnVertices = 0;

    JSONParser parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE);
    while (value.hasNext()) {
        Text vrecString = value.next();
        JSONObject obj;/*from   ww w  .j av  a2s  .  c  om*/
        try {
            obj = (JSONObject) parser.parse(vrecString.toString());
            int owner = ((Long) obj.get("owner")).intValue();
            if (owner == key.get()) {
                numOwnVertices++;
                reporter.incrCounter(COUNTER.OWN_VERTICES, 1);
            }
            out.collect(new Text("partition" + key.get() + " vrecord"), vrecString);
            numVertices++;
            reporter.incrCounter(COUNTER.VERTICES, 1);
        } catch (ParseException e) {
            e.printStackTrace();
        } // end try parsing
    } // end while

    JSONObject summary = new JSONObject();
    summary.put("numVertices", numVertices);
    summary.put("numOwnVertices", numOwnVertices);
    out.collect(new Text("partition" + key.get() + " meta"), new Text(summary.toJSONString()));
}

From source file:com.j.distributed.counter.CounterReducer.java

@Override
public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
    int i = 0;/*from   w w  w  .jav a2s . c o  m*/
    for (IntWritable value : values)
        i += value.get();
    context.write(key, new IntWritable(i));
}

From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java

License:Apache License

public void writeRawInt(IntWritable iw) throws IOException {
    out.writeRawInt(iw.get());
}

From source file:com.jfolson.hive.serde.RTypedBytesWritableOutput.java

License:Apache License

public void writeInt(IntWritable iw) throws IOException {
    out.writeInt(iw.get());
}

From source file:com.jhkt.playgroundArena.hadoop.tasks.jobs.reducer.CountReducer.java

License:Apache License

protected void reduce(Text key, Iterable<IntWritable> values,
        Reducer<Text, IntWritable, Text, IntWritable>.Context context)
        throws IOException, InterruptedException {
    int count = 0;

    for (IntWritable value : values) {
        count += value.get();
    }//from w  w w.j  a  v  a  2s.c o m
    context.write(key, new IntWritable(count));
}

From source file:com.kit.udf.UDFSubstrForOracle.java

License:Apache License

public Text evaluate(Text t, IntWritable pos, IntWritable len) {

    if ((t == null) || (pos == null) || (len == null)) {
        return null;
    }/*from w  w w.  ja v  a  2  s . c o m*/

    r.clear();
    if ((len.get() <= 0)) {
        //return r;
        return null;
    }

    String s = t.toString();
    if ((Math.abs(pos.get()) > s.length())) {
        //return r;
        return null;
    }

    int start, end;

    if (pos.get() > 0) {
        start = pos.get() - 1;
    } else if (pos.get() < 0) {
        start = s.length() + pos.get();
    } else {
        start = 0;
    }

    if ((s.length() - start) < len.get()) {
        end = s.length();
    } else {
        end = start + len.get();
    }

    r.set(s.substring(start, end));
    return r;
}

From source file:com.kylinolap.job.hadoop.cardinality.ColumnCardinalityReducer.java

License:Apache License

@Override
public void reduce(IntWritable key, Iterable<BytesWritable> values, Context context)
        throws IOException, InterruptedException {
    for (BytesWritable v : values) {
        int skey = key.get();
        ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
        HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
        hll.readRegisters(buffer);//w w w.j ava 2s .c  o m
        getHllc(skey).merge(hll);
        hll.clear();
    }
}

From source file:com.linkedin.cubert.io.CompactDeserializer.java

License:Open Source License

@SuppressWarnings("unchecked")
@Override/*  w  w w.  ja  v  a 2s. co  m*/
public K deserialize(K object) throws IOException {
    if (in.available() == 0)
        throw new EOFException();

    Tuple t = (Tuple) object;

    if (t == null || t.size() != datatypes.length)
        t = TupleFactory.getInstance().newTuple(datatypes.length);

    for (int i = 0; i < datatypes.length; i++) {
        Object field = null;

        switch (datatypes[i]) {
        case BOOLEAN: {
            IntWritable writable = VariableLengthEncoder.decodeInteger(in);
            field = (writable == null) ? null : (Boolean) (writable.get() == 1);
            break;
        }
        case BYTE: {
            IntWritable writable = VariableLengthEncoder.decodeInteger(in);
            field = (writable == null) ? null : (byte) writable.get();
            break;
        }
        case DOUBLE: {
            DoubleWritable writable = VariableLengthEncoder.decodeDouble(in);
            field = (writable == null) ? null : writable.get();
            break;
        }
        case FLOAT: {
            FloatWritable writable = VariableLengthEncoder.decodeFloat(in);
            field = (writable == null) ? null : writable.get();
            break;
        }
        case INT: {
            IntWritable writable = VariableLengthEncoder.decodeInteger(in);
            field = (writable == null) ? null : writable.get();
            break;
        }
        case LONG: {
            LongWritable writable = VariableLengthEncoder.decodeLong(in);
            field = (writable == null) ? null : writable.get();
            break;
        }
        case STRING: {
            IntWritable writable = VariableLengthEncoder.decodeInteger(in);
            if (writable == null)
                field = null;
            else {
                int length = writable.get();
                while (length > buffer.length)
                    buffer = new byte[2 * buffer.length];

                in.read(buffer, 0, length);
                field = new String(buffer, 0, length);
            }
            break;
        }
        default:
            throw new RuntimeException("Cannot deserialize column of type " + datatypes[i]);
        }

        t.set(i, field);
    }

    return (K) t;
}