Example usage for org.apache.hadoop.io Text getLength

List of usage examples for org.apache.hadoop.io Text getLength

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. getLength

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text getLength.

Prototype

@Override
public int getLength()

Source Link

Document

Returns the number of bytes in the byte array

Usage

From source file:org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.java

License:Apache License

public void init(Text key, byte typeId) {
    this.typeId = typeId;
    this.rawKey = key;
    if (isNumberFamily()) {
        String valueStr = new String(key.getBytes(), 1, key.getLength() - 1);
        if (isIntegerFamily()) {
            this.keyInObj = Long.parseLong(valueStr);
        } else {/*from   w ww .jav a  2s  .  c o  m*/
            this.keyInObj = Double.parseDouble(valueStr);
        }
    } else {
        this.keyInObj = key;
    }
}

From source file:org.apache.kylin.engine.mr.steps.UHCDictionaryMapper.java

License:Apache License

@Override
public void doMap(NullWritable key, Text value, Context context) throws IOException, InterruptedException {
    tmpBuf.clear();//  w ww  .j  a v  a2s  .c o m
    int size = value.getLength() + 1;
    if (size >= tmpBuf.capacity()) {
        tmpBuf = ByteBuffer.allocate(countNewSize(tmpBuf.capacity(), size));
    }
    tmpBuf.put(Bytes.toBytes(index)[3]);
    tmpBuf.put(value.getBytes(), 0, value.getLength());
    outputKey.set(tmpBuf.array(), 0, tmpBuf.position());

    sortableKey.init(outputKey, type);
    context.write(sortableKey, NullWritable.get());
}

From source file:org.apache.kylin.engine.mr.steps.UHCDictionaryReducer.java

License:Apache License

@Override
public void doReduce(SelfDefineSortableKey skey, Iterable<NullWritable> values, Context context)
        throws IOException, InterruptedException {
    Text key = skey.getText();
    String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
    builder.addValue(value);// w  w  w . j  av a  2 s .c o  m
}

From source file:org.apache.kylin.engine.spark.SparkUtil.java

License:Apache License

public static JavaRDD<String[]> hiveRecordInputRDD(boolean isSequenceFile, JavaSparkContext sc,
        String inputPath, String hiveTable) {
    JavaRDD<String[]> recordRDD;

    if (isSequenceFile) {
        recordRDD = sc.sequenceFile(inputPath, BytesWritable.class, Text.class).values()
                .map(new Function<Text, String[]>() {
                    @Override/*from  w  w  w .  j a  v  a2  s. com*/
                    public String[] call(Text text) throws Exception {
                        String s = Bytes.toString(text.getBytes(), 0, text.getLength());
                        return s.split(BatchConstants.SEQUENCE_FILE_DEFAULT_DELIMITER);
                    }
                });
    } else {
        SparkSession sparkSession = SparkSession.builder().config(sc.getConf()).enableHiveSupport()
                .getOrCreate();
        final Dataset intermediateTable = sparkSession.table(hiveTable);
        recordRDD = intermediateTable.javaRDD().map(new Function<Row, String[]>() {
            @Override
            public String[] call(Row row) throws Exception {
                String[] result = new String[row.size()];
                for (int i = 0; i < row.size(); i++) {
                    final Object o = row.get(i);
                    if (o != null) {
                        result[i] = o.toString();
                    } else {
                        result[i] = null;
                    }
                }
                return result;
            }
        });
    }

    return recordRDD;
}

From source file:org.apache.kylin.job.hadoop.cube.BaseCuboidMapperTest.java

License:Apache License

@Test
public void testMapperWithHeader() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"),
            new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());//from   ww  w .  j a  va2  s  .co  m
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]",
            decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33");
}

From source file:org.apache.kylin.job.hadoop.cube.BaseCuboidMapperTest.java

License:Apache License

@Test
public void testMapperWithNull() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"),
            new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());//  w  w w .j  av a2  s  .  co  m
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]",
            decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0");
}

From source file:org.apache.kylin.job.hadoop.cube.CubeHFileMapper2Test.java

License:Apache License

@Test
public void testBasic() throws Exception {

    Configuration hconf = new Configuration();
    Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV);

    CubeHFileMapper mapper = new CubeHFileMapper();
    mapper.setup(context);//from   w ww.j  av  a 2  s .c o m

    Text key = new Text("not important");
    Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });

    mapper.map(key, value, context);

    ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
    KeyValue outValue = (KeyValue) outKV[1];

    assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(),
            outKey.getLength()) == 0);

    assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(),
            outValue.getValueOffset(), outValue.getValueLength()) == 0);
}

From source file:org.apache.kylin.storage.hbase.steps.CubeHFileMapper.java

License:Apache License

@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
    outputKey.set(key.getBytes(), 0, key.getLength());
    KeyValue outputValue;/*www  .j a  v a 2s . com*/

    int n = keyValueCreators.size();
    if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for simple full copy

        outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength());
        context.write(outputKey, outputValue);

    } else { // normal (complex) case that distributes measures to multiple HBase columns

        inputCodec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), inputMeasures);

        for (int i = 0; i < n; i++) {
            outputValue = keyValueCreators.get(i).create(key, inputMeasures);
            context.write(outputKey, outputValue);
        }
    }
}

From source file:org.apache.kylin.storage.hbase.steps.CubeHFileMapper2Test.java

License:Apache License

@Test
public void testBasic() throws Exception {

    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    Context context = MockupMapContext.create(hconf, cubeName, outKV);

    CubeHFileMapper mapper = new CubeHFileMapper();
    mapper.setup(context);/*from   w  w w  .  j  a  v  a  2 s  .  c  o  m*/

    Text key = new Text("not important");
    Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });

    mapper.map(key, value, context);

    ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
    KeyValue outValue = (KeyValue) outKV[1];

    assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(),
            outKey.getLength()) == 0);

    assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(),
            outValue.getValueOffset(), outValue.getValueLength()) == 0);
}

From source file:org.apache.kylin.storage.hbase.steps.KeyValueCreator.java

License:Apache License

public KeyValue create(Text key, Object[] measureValues) {
    return create(key.getBytes(), 0, key.getLength(), measureValues);
}