Example usage for org.apache.hadoop.io Text getBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text getBytes.

Prototype

@Override
public byte[] getBytes()

Source Link

Document

Returns the raw bytes; however, only data up to #getLength() is valid.

Usage

From source file:com.inmobi.messaging.consumer.databus.mapreduce.DatabusRecordReader.java

License:Apache License

@Override
public Message getCurrentValue() throws IOException, InterruptedException {
    Text text = lineReader.getCurrentValue();
    // get the byte array corresponding to the value read
    int length = text.getLength();
    byte[] msg = new byte[length];
    System.arraycopy(text.getBytes(), 0, msg, 0, length);
    return DatabusUtil.decodeMessage(msg);
}

From source file:com.kasabi.labs.freebase.mr.Freebase2RDFReducer.java

License:Apache License

@Override
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
    for (Text value : values) {
        log.debug("< ({}, {})", key, value);
        k.clear();/*from   w  ww. j  a  v a 2  s  . c o m*/
        byte[] kb = key.getBytes();
        k.append(kb, 0, key.getLength());
        byte[] vb = value.toString().getBytes();
        k.append(vb, 0, vb.length);
        context.write(k, nullWritable);
        log.debug("> ({}, {})", k, nullWritable);
    }
}

From source file:com.kylinolap.cube.common.BytesSplitter.java

License:Apache License

public int detectDelim(Text value, int expectedParts) {
    for (int i = 0; i < COMMON_DELIMS.length; i++) {
        int nParts = split(value.getBytes(), value.getLength(), (byte) COMMON_DELIMS[i]);
        if (nParts == expectedParts)
            return COMMON_DELIMS[i];
    }/* www. jav a 2 s .c om*/
    throw new RuntimeException("Cannot detect delimeter from first line -- " + value.toString() + " -- expect "
            + expectedParts + " columns");
}

From source file:com.kylinolap.cube.measure.MeasureCodec.java

License:Apache License

public void decode(Text bytes, Object[] result) {
    decode(ByteBuffer.wrap(bytes.getBytes(), 0, bytes.getLength()), result);
}

From source file:com.kylinolap.job.hadoop.cube.BaseCuboidMapper.java

License:Apache License

@Override
public void map(KEYIN key, Text value, Context context) throws IOException, InterruptedException {
    counter++;//ww  w . ja v a2 s .c  om
    if (counter % BatchConstants.COUNTER_MAX == 0) {
        logger.info("Handled " + counter + " records!");
    }

    try {
        bytesSplitter.split(value.getBytes(), value.getLength(), byteRowDelimiter);
        intermediateTableDesc.sanityCheck(bytesSplitter);

        byte[] rowKey = buildKey(bytesSplitter.getSplitBuffers());
        outputKey.set(rowKey, 0, rowKey.length);

        buildValue(bytesSplitter.getSplitBuffers());
        outputValue.set(valueBuf.array(), 0, valueBuf.position());

        context.write(outputKey, outputValue);
    } catch (Exception ex) {
        handleErrorRecord(bytesSplitter, ex);
    }
}

From source file:com.kylinolap.job.hadoop.cube.BaseCuboidMapperTest.java

License:Apache License

@Test
public void testMapperWithHeader() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"),
            new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.331"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());//from  w w w. j  av  a 2 s .  co  m
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]",
            decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33",
            1);
}

From source file:com.kylinolap.job.hadoop.cube.BaseCuboidMapperTest.java

License:Apache License

@Test
public void testMapperWithNull() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"),
            new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N\\N"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());//from w  w  w  .j a v a  2 s  .c o m
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]",
            decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", 1L);
}

From source file:com.kylinolap.job.hadoop.cube.CubeHFileMapper.java

License:Apache License

@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    outputKey.set(key.getBytes(), 0, key.getLength());
    KeyValue outputValue;//from ww  w  .ja va2 s.com

    int n = keyValueCreators.size();
    if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for
                                                        // simple full copy

        outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength());
        context.write(outputKey, outputValue);

    } else { // normal (complex) case that distributes measures to multiple
             // HBase columns

        inputCodec.decode(value, inputMeasures);

        for (int i = 0; i < n; i++) {
            outputValue = keyValueCreators.get(i).create(key, inputMeasures);
            context.write(outputKey, outputValue);
        }
    }
}

From source file:com.kylinolap.job.hadoop.cube.CubeHFileMapper2Test.java

License:Apache License

@Test
public void testBasic() throws Exception {

    Configuration hconf = new Configuration();
    Context context = MockupMapContext.create(hconf, this.getTestConfig().getMetadataUrl(), cubeName, outKV);

    CubeHFileMapper mapper = new CubeHFileMapper();
    mapper.setup(context);/*from  www  .  jav  a  2 s. c o m*/

    Text key = new Text("not important");
    Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });

    mapper.map(key, value, context);

    ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
    KeyValue outValue = (KeyValue) outKV[1];

    assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(),
            outKey.getLength()) == 0);

    assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(),
            outValue.getValueOffset(), outValue.getValueLength()) == 0);
}

From source file:com.kylinolap.job.hadoop.cube.FactDistinctColumnsCombiner.java

License:Apache License

@Override
public void reduce(ShortWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

    HashSet<ByteArray> set = new HashSet<ByteArray>();
    for (Text textValue : values) {
        ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
        set.add(value);//from   ww  w  . j a  va  2  s.co m
    }

    for (ByteArray value : set) {
        outputValue.set(value.data);
        context.write(key, outputValue);
    }
}