List of usage examples for org.apache.hadoop.io Text set
public void set(byte[] utf8, int start, int len)
From source file:org.apache.kylin.engine.mr.steps.CubeReducerTest.java
License:Apache License
private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) { Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) }; ByteBuffer buf = codec.encode(values); Text t = new Text(); t.set(buf.array(), 0, buf.position()); return t;//w w w .j a va2s .c o m }
From source file:org.apache.kylin.engine.mr.steps.NumberDictionaryForestTest.java
License:Apache License
private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); keyBuffer.put(Bytes.toBytes(partationId)[3]); keyBuffer.put(Bytes.toBytes(str)); //System.out.println(Arrays.toString(keyBuffer.array())); byte[] valueField = Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1); //System.out.println("new string:"+new String(valueField)); //System.out.println("arrays toString:"+Arrays.toString(valueField)); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); sortableKey.init(outputKey, typeFlag); keyList.add(sortableKey);/*www . ja va2 s .c o m*/ } return keyList; }
From source file:org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyTest.java
License:Apache License
private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); keyBuffer.put(Bytes.toBytes(partationId)[3]); keyBuffer.put(Bytes.toBytes(str)); Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); sortableKey.init(outputKey, typeFlag); keyList.add(sortableKey);//from w ww . ja v a 2 s .co m } return keyList; }
From source file:org.apache.kylin.storage.hbase.steps.SequenceFileCuboidWriter.java
License:Apache License
@Override protected void writeAsKeyValue(ByteArrayWritable key, ByteArrayWritable value) throws IOException { Text outputValue = new Text(); Text outputKey = new Text(); outputKey.set(key.array(), key.offset(), key.length()); outputValue.set(value.array(), value.offset(), value.length()); writer.append(outputKey, outputValue); }
From source file:org.apache.orc.mapred.OrcMapredRecordReader.java
License:Apache License
static Text nextString(ColumnVector vector, int row, Object previous) { if (vector.isRepeating) { row = 0;/*from w ww . jav a 2s .c o m*/ } if (vector.noNulls || !vector.isNull[row]) { Text result; if (previous == null || previous.getClass() != Text.class) { result = new Text(); } else { result = (Text) previous; } BytesColumnVector bytes = (BytesColumnVector) vector; result.set(bytes.vector[row], bytes.start[row], bytes.length[row]); return result; } else { return null; } }
From source file:org.gradoop.flink.io.impl.tlf.inputformats.TLFRecordReader.java
License:Apache License
/** * Reads the next key/value pair from the input for processing. * * @param key the new key//from w ww . j av a2s . co m * @param value the new value * @return true if a key/value pair was found * @throws IOException */ private boolean next(LongWritable key, Text value) throws IOException { if (fsin.getPos() < end && readUntilMatch(TLFConstants.START_TAG.getBytes(Charsets.UTF_8), false)) { try { buffer.write(TLFConstants.START_TAG.getBytes(Charsets.UTF_8)); if (readUntilMatch(TLFConstants.END_TAG.getBytes(Charsets.UTF_8), true)) { key.set(fsin.getPos()); if (fsin.getPos() != end) { //- end tag because it is the new start tag and shall not be added valueLength = buffer.getLength() - TLFConstants.END_TAG.getBytes(Charsets.UTF_8).length; } else { // in this case there is no new start tag valueLength = buffer.getLength(); } //- end tag because it is the new start tag and shall not be added value.set(buffer.getData(), 0, valueLength); //set the buffer to position before end tag of old graph which is // start tag of the new one fsin.seek(fsin.getPos() - TLFConstants.END_TAG.getBytes(Charsets.UTF_8).length); return true; } } finally { buffer.reset(); } } return false; }
From source file:org.gradoop.io.impl.tlf.inputformats.TLFRecordReader.java
License:Open Source License
/** * Reads the next key/value pair from the input for processing. * * @param key the new key//from ww w . j ava 2 s. co m * @param value the new value * @return true if a key/value pair was found * @throws IOException */ private boolean next(LongWritable key, Text value) throws IOException { if (fsin.getPos() < end && readUntilMatch(START_TAG_BYTE, false)) { try { buffer.write(START_TAG_BYTE); if (readUntilMatch(END_TAG_BYTE, true)) { key.set(fsin.getPos()); if (fsin.getPos() != end) { //- end tag because it is the new start tag and shall not be added valueLength = buffer.getLength() - END_TAG_BYTE.length; } else { // in this case there is no new start tag valueLength = buffer.getLength(); } //- end tag because it is the new start tag and shall not be added value.set(buffer.getData(), 0, valueLength); //set the buffer to position before end tag of old graph which is // start tag of the new one fsin.seek(fsin.getPos() - END_TAG_BYTE.length); return true; } } finally { buffer.reset(); } } return false; }
From source file:org.mitre.bio.mapred.io.FastaRecordReader.java
License:Open Source License
/** * Reads the next key/value pair from the input for processing. * * @param key the key to read data into/*from w ww. jav a 2 s . c om*/ * @param value the value to read data into * @return true iff a key/value was read, false if at EOF */ @Override public synchronized boolean next(LongWritable key, Text value) throws IOException { this.buffer.reset(); if (this.pos < this.end) { try { // Find the being of a new record block if (readLinesUntilStartsWithMatch(startTag, false)) { // Read until we find the endTag or EOF readLinesBeforeStartsWithMatch(startTag, true); if (buffer.size() > 0) { key.set(this.pos); value.set(buffer.getData(), 0, buffer.getLength()); return true; } } } finally { LOG.debug("Uncaught exception!"); this.buffer.reset(); } } return false; }
From source file:wikiParser.mapReduce.util.KeyValueLineRecordReader.java
License:Apache License
public static void setKeyValue(Text key, Text value, byte[] line, int lineLen, int pos) { if (pos == -1) { key.set(line, 0, lineLen); value.set(""); } else {//from w ww. ja v a 2 s. c om key.set(line, 0, pos); value.set(line, pos + 1, lineLen - pos - 1); } }