Example usage for org.apache.hadoop.io LongWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io LongWritable get.

Prototype

public long get() 

Source Link

Document

Return the value of this LongWritable.

Usage

From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java

License:Apache License

public Message(LongWritable triangleA, LongWritable triangleB, LongWritable triangleC) {
    this.type = Type.TRIANGLE;
    this.triangleA = new LongWritable(triangleA.get());
    this.triangleB = new LongWritable(triangleB.get());
    this.triangleC = new LongWritable(triangleC.get());
}

---

From source file:com.cloudera.sa.giraph.examples.ktrusses.Message.java

License:Apache License

public Message(LongWritable trussID) {
    this.type = Type.TRUSS_ID;
    this.trussID = new LongWritable(trussID.get());
}

---

From source file:com.conversantmedia.mapreduce.example.avro.AvroWordCountReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;/*from ww  w. jav  a  2s .c  om*/
    for (LongWritable value : values) {
        sum += value.get();
    }
    AvroExample.Builder builder = AvroExample.newBuilder();
    builder.setWord(key.toString());
    builder.setFrequency(sum);
    AvroExample datum = builder.build();
    aKey.datum(datum);

    context.write(aKey, NullWritable.get());
    avroMultiOut.write(aKey, NullWritable.get(), "avroMulti");
}

---

From source file:com.conversantmedia.mapreduce.example.WordCountReducer.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;/*ww  w. j  a va  2s  .  c o m*/
    for (LongWritable value : values) {
        sum += value.get();
    }
    context.write(key, new LongWritable(sum));
}

---

From source file:com.conversantmedia.mapreduce.example.WordCountReducerWithMinimum.java

License:Apache License

@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
        throws IOException, InterruptedException {
    int sum = 0;/*from   w w  w  . java  2  s  .  c om*/
    for (LongWritable value : values) {
        sum += value.get();
    }

    // Ensure we're above the minimum
    if (sum >= minimum) {
        context.write(key, new LongWritable(sum));
    }
}

---

From source file:com.datasalt.utils.mapred.counter.TestMapRedCounter.java

License:Apache License

/**
 * Return a map with the counts for the items. Key: [typeIdentifier]:[group]:[item] 
 *///w  ww  .  j  a va 2s  .  co  m
private HashMap<String, Long> itemCountAsMap(FileSystem fs, String file) throws IOException {
    HashMap<String, Long> m = new HashMap<String, Long>();

    SequenceFile.Reader r = new SequenceFile.Reader(getFs(), new Path(file), getConf());

    CounterKey key = new CounterKey();
    LongWritable count = new LongWritable();
    while (r.next(key)) {
        r.getCurrentValue(count);
        m.put(key.getGroupId() + ":" + getSer().deser(new Text(), key.getGroup()) + ":"
                + getSer().deser(new Text(), key.getItem()), count.get());
    }

    return m;
}

---

From source file:com.datascience.cascading.scheme.CsvScheme.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall)
        throws TapException, IOException {
    Object[] context = sourceCall.getContext();
    if (!sourceCall.getInput().next(context[0], context[1])) {
        return false;
    }/* w  w  w .j  a  va 2s .  c o m*/

    TupleEntry entry = sourceCall.getIncomingEntry();

    ListWritable<Text> values = (ListWritable<Text>) context[1];

    Fields fields = getSourceFields();
    if ((format.getHeader() != null) && !areFieldsInFormatHeaders(fields)) {
        try {
            LongWritable pos = (LongWritable) context[0];
            Long position = pos.get();
            String message = String.format("%s: %s",
                    "Failed to parse record. fields not in header record at position: ", position);
            LOGGER.warn(message);
            if (strict) {
                throw new CsvParseException(message);
            } else {
                return true;
            }
        } catch (CsvParseException e) {
            throw new TapException(e);
        }
    }
    int check = strict ? fields.size() : values.size() != fields.size() ? values.size() : fields.size();
    int checkDiff = check - fields.size();
    for (int i = 0; i < (checkDiff < 1 ? fields.size() : values.size()); i++) {
        int index = indices != null && checkDiff < 1 ? indices.get(fields.get(i).toString()) : i;

        //fill empty values with null for records missing values
        Text value = values.size() - i < 1 ? null : values.get(index);

        if (value == null) {
            entry.setString(i, null);
        } else {
            try {
                entry.setString(i, value.toString());
            } catch (Exception e) {
                if (!strict) {
                    Tuple tuple = new Tuple();
                    for (Text val : values) {
                        tuple.addString(val.toString());
                    }
                    throw new TapException(e.getMessage(), e, tuple);
                } else {
                    return false;
                }
            }
        }
    }
    return true;
}

---

From source file:com.facebook.hive.orc.lazy.OrcLazyLongObjectInspector.java

License:Open Source License

@Override
public Object getPrimitiveJavaObject(Object o) {
    LongWritable writable = (LongWritable) getPrimitiveWritableObject(o);
    return writable == null ? null : Long.valueOf(writable.get());
}

---

From source file:com.facebook.hive.orc.TestOrcFile.java

License:Apache License

private void compareRowsWithoutNextIsNull(OrcStruct row, RandomRowInputs inputs, int rowNumber,
        NumberOfNulls numNulls, boolean usingPrimitives) throws Exception {

    ReallyBigRow expected = null;/*from w w  w  . j a v  a 2 s  . c  o m*/
    switch (numNulls) {
    case MANY:
    case SOME:
        expected = createRandomRowWithNulls(inputs.intValues, inputs.doubleValues, inputs.stringValues,
                inputs.byteValues, inputs.words, rowNumber, numNulls);
        break;
    case NONE:
        expected = createRandomRow(inputs.intValues, inputs.doubleValues, inputs.stringValues,
                inputs.byteValues, inputs.words, rowNumber);
        break;
    }

    OrcLazyBoolean lazyBoolean1 = (OrcLazyBoolean) row.getFieldValue(0);
    BooleanWritable boolean1 = (BooleanWritable) lazyBoolean1.materialize();
    if (boolean1 == null) {
        assertNull(expected.boolean1);
    } else {
        assertEquals(expected.boolean1.booleanValue(), boolean1.get());
    }

    ByteWritable byte1 = (ByteWritable) ((OrcLazyByte) row.getFieldValue(1)).materialize();
    if (byte1 == null) {
        assertNull(expected.byte1);
    } else {
        assertEquals(expected.byte1.byteValue(), byte1.get());
    }

    OrcLazyShort lazyShort1 = (OrcLazyShort) row.getFieldValue(2);
    ShortWritable short1 = (ShortWritable) lazyShort1.materialize();
    if (short1 == null) {
        assertNull(expected.short1);
    } else {
        assertEquals(expected.short1.shortValue(), short1.get());
    }

    OrcLazyInt lazyInt1 = (OrcLazyInt) row.getFieldValue(3);
    IntWritable int1 = (IntWritable) lazyInt1.materialize();
    if (int1 == null) {
        assertNull(expected.int1);
    } else {
        assertEquals(expected.int1.intValue(), int1.get());
    }

    OrcLazyLong lazyLong1 = (OrcLazyLong) row.getFieldValue(4);
    LongWritable long1 = (LongWritable) lazyLong1.materialize();
    if (long1 == null) {
        assertNull(expected.long1);
    } else {
        assertEquals(expected.long1.longValue(), long1.get());
    }

    OrcLazyShort lazyShort2 = (OrcLazyShort) row.getFieldValue(5);
    ShortWritable short2 = (ShortWritable) lazyShort2.materialize();
    if (short2 == null) {
        assertNull(expected.short2);
    } else {
        assertEquals(expected.short2.shortValue(), short2.get());
    }

    OrcLazyInt lazyInt2 = (OrcLazyInt) row.getFieldValue(6);
    IntWritable int2 = (IntWritable) lazyInt2.materialize();
    if (int2 == null) {
        assertNull(expected.int2);
    } else {
        assertEquals(expected.int2.intValue(), int2.get());
    }

    OrcLazyLong lazyLong2 = (OrcLazyLong) row.getFieldValue(7);
    LongWritable long2 = (LongWritable) lazyLong2.materialize();
    if (long2 == null) {
        assertNull(expected.long2);
    } else {
        assertEquals(expected.long2.longValue(), long2.get());
    }

    OrcLazyShort lazyShort3 = (OrcLazyShort) row.getFieldValue(8);
    ShortWritable short3 = (ShortWritable) lazyShort3.materialize();
    if (short3 == null) {
        assertNull(expected.short3);
    } else {
        assertEquals(expected.short3.shortValue(), short3.get());
    }

    OrcLazyInt lazyInt3 = (OrcLazyInt) row.getFieldValue(9);
    IntWritable int3 = (IntWritable) lazyInt3.materialize();
    if (int3 == null) {
        assertNull(expected.int3);
    } else {
        assertEquals(expected.int3.intValue(), int3.get());
    }

    OrcLazyLong lazyLong3 = (OrcLazyLong) row.getFieldValue(10);
    LongWritable long3 = (LongWritable) lazyLong3.materialize();
    if (long3 == null) {
        assertNull(expected.long3);
    } else {
        assertEquals(expected.long3.longValue(), long3.get());
    }

    OrcLazyFloat lazyFloat1 = (OrcLazyFloat) row.getFieldValue(11);
    FloatWritable float1 = (FloatWritable) lazyFloat1.materialize();
    if (float1 == null) {
        assertNull(expected.float1);
    } else {
        assertEquals(expected.float1.floatValue(), float1.get(), 0.0001);
    }

    OrcLazyDouble lazyDouble1 = (OrcLazyDouble) row.getFieldValue(12);
    DoubleWritable double1 = (DoubleWritable) lazyDouble1.materialize();
    if (double1 == null) {
        assertNull(expected.double1);
    } else {
        assertEquals(expected.double1.doubleValue(), double1.get(), 0.0001);
    }

    BytesWritable bytes1 = (BytesWritable) ((OrcLazyBinary) row.getFieldValue(13)).materialize();
    if (bytes1 == null) {
        assertNull(expected.bytes1);
    } else {
        assertEquals(expected.bytes1, bytes1);
    }

    Text string1 = (Text) ((OrcLazyString) row.getFieldValue(14)).materialize();
    if (string1 == null) {
        assertNull(expected.string1);
    } else {
        assertEquals(expected.string1, string1);
    }

    Text string2 = (Text) ((OrcLazyString) row.getFieldValue(15)).materialize();
    if (string2 == null) {
        assertNull(expected.string2);
    } else {
        assertEquals(expected.string2, string2);
    }

    Text string3 = (Text) ((OrcLazyString) row.getFieldValue(16)).materialize();
    if (string3 == null) {
        assertNull(expected.string3);
    } else {
        assertEquals(expected.string3, string3);
    }

    OrcStruct middle = (OrcStruct) ((OrcLazyStruct) row.getFieldValue(17)).materialize();
    if (middle == null) {
        assertNull(expected.middle);
    } else {
        final List<InnerStruct> expectedList = expected.middle.list;
        final List<OrcStruct> actualList = (List) middle.getFieldValue(0);
        compareListOfStructs(expectedList, actualList);
        final List<String> actualFieldNames = middle.getFieldNames();
        final List<String> expectedFieldNames = ImmutableList.of("list");
        compareLists(expectedFieldNames, actualFieldNames);
    }

    List list = (List) ((OrcLazyList) row.getFieldValue(18)).materialize();
    if (list == null) {
        assertNull(expected.list);
    } else {
        compareListOfStructs(expected.list, list);
    }

    Map map = (Map) ((OrcLazyMap) row.getFieldValue(19)).materialize();
    if (map == null) {
        assertNull(expected.map);
    } else {
        compareMap(expected.map, map);
    }

    if (usingPrimitives) {
        compareRowsUsingPrimitives(expected, lazyBoolean1, lazyShort1, lazyInt1, lazyLong1, lazyShort2,
                lazyInt2, lazyLong2, lazyShort3, lazyInt3, lazyLong3, lazyFloat1, lazyDouble1);
    }
}

---

From source file:com.facebook.hiveio.mapreduce.output.HiveTools.java

License:Apache License

/**
 * Map hive record// w  w w  .  java 2  s  . co m
 *
 * @param conf Configuration
 * @param value data
 * @return hive record
 */
public static HiveWritableRecord mapToHiveRecord(Configuration conf, MapWritable value) {
    HiveTableSchema schema = HiveTableSchemas.lookup(conf, getHiveTableName());
    HiveWritableRecord record = HiveRecordFactory.newWritableRecord(schema);
    for (Map.Entry<Writable, Writable> entry : value.entrySet()) {
        IntWritable intKey = (IntWritable) entry.getKey();
        LongWritable longValue = (LongWritable) entry.getValue();
        record.set(intKey.get(), longValue.get());
    }
    return record;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »