List of usage examples for org.apache.hadoop.io DataInputBuffer DataInputBuffer
public DataInputBuffer()
From source file:com.datasalt.pangool.io.TestBitField.java
License:Apache License
static void serDeCheck(BitField bf) throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); DataInputBuffer dib = new DataInputBuffer(); bf.ser(dob);//w w w . j a v a2 s. co m dib.reset(dob.getData(), dob.getData().length); // Checking DataInput deser BitField newbf = new BitField(); newbf.set(1000); newbf.deser(dib); assertEquals(bf.toString() + " vs " + newbf.toString(), 0, bf.compareTo(newbf)); // Asserting that the rightmost bit is always 0. Important for comparisons byte[] arr = newbf.getBackingArray(); for (int i = 0; i < arr.length; i++) { assertEquals(0, arr[i] & 1); } //Checking byte array deserialization newbf = new BitField(); newbf.set(1000); newbf.deser(dob.getData(), 0); assertEquals(bf.toString() + " vs " + newbf.toString(), 0, bf.compareTo(newbf)); // Asserting that the rightmost bit is always 0. Important for comparisons arr = newbf.getBackingArray(); for (int i = 0; i < arr.length; i++) { assertEquals(0, arr[i] & 1); } }
From source file:com.datasalt.pangool.io.TestBitField.java
License:Apache License
static void checkReturnOfDeser(int bits) throws IOException { BitField bf = new BitField(); bf.set(bits - 1);/*from w ww . ja v a 2 s .co m*/ DataOutputBuffer dob = new DataOutputBuffer(); DataInputBuffer dib = new DataInputBuffer(); bf.ser(dob); dib.reset(dob.getData(), dob.getData().length); // Checking DataInput deser BitField newbf = new BitField(); int read = newbf.deser(dib); assertEquals("For bits " + bits, ((bits - 1) / 7) + 1, read); //Checking byte array deserialization newbf = new BitField(); read = newbf.deser(dob.getData(), 0); assertEquals("For bits " + bits, ((bits - 1) / 7) + 1, read); }
From source file:com.ebay.erl.mobius.core.datajoin.DataJoinKey.java
License:Apache License
@Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { DataInputBuffer d1 = new DataInputBuffer(); d1.reset(b1, s1, l1);/*from www .java2 s . c om*/ DataInputBuffer d2 = new DataInputBuffer(); d2.reset(b2, s2, l2); int _compare_result = Integer.MAX_VALUE; try { // the comparing ordering: // 1. DataJoinKey#KEY_FIELDNAME // 2. DataJoinKey#DATASET_ID_FIELDNAME // 3. DataJoinKey#SORT_KEYWORD_FIELDNAME - removed // 4. DataJoinKey#SORT_COMPARATOR_FIELDNAME - removed // read number of columns from the two tuple, // but there is no need to compare the length // of columns, we just read the values. d1.readInt(); d2.readInt(); ////////////////////////////////////////////////////////// // compare KEY, values from DataJoinKey#KEY_FIELDNAME // KEY represents the actual key user specified /////////////////////////////////////////////////////////// byte type1 = d1.readByte(); byte type2 = d2.readByte(); _COLUMN_COMPARATOR.setType(type1, type2); // writable, check if they are Tuple or NullWritable if (type1 == Tuple.NULL_WRITABLE_TYPE && type2 == Tuple.NULL_WRITABLE_TYPE) { // consider equal, do nothing _compare_result = 0; } else if (type1 == Tuple.TUPLE_TYPE && type2 == Tuple.TUPLE_TYPE) { // both are Tuple Tuple k1 = (Tuple) getKey(type1, d1); Tuple k2 = (Tuple) getKey(type2, d2); _compare_result = _COLUMN_COMPARATOR.compareKey(k1, k2, this.getSorter(), conf); } else { // DataJoinKey only support NullWritable and Tuple for the DataJoinKey#KEY_FIELDNAME throw new IllegalArgumentException( "Cannot compare " + Tuple.getTypeString(type1) + " and " + Tuple.getTypeString(type2)); } // if they are not the same, these two records should go to // different reducer, or different reduce iteration. if (_compare_result != 0) return _compare_result; ////////////////////////////////////////////////////////////////////////// // compare DATASET_ID, values from DataJoinKey#DATASET_ID_FIELDNAME, // at this point, the keys are the same, they should go to the same // reducer, we need to make sure the values from DATASET1 always come // before DATASET2, so we need to compare the DATASET_ID here. ////////////////////////////////////////////////////////////////////////// try { _COLUMN_COMPARATOR.setType(d1.readByte(), d2.readByte()); _compare_result = _COLUMN_COMPARATOR.compare(d1, d2, this.conf); if (_compare_result != 0) return _compare_result; } catch (IOException e) { byte[] b = new byte[l1]; for (int i = 0; i < l1; i++) { b[i] = b1[s1 + i]; } System.err.println(Arrays.toString(b)); System.err.println("type1:" + type1 + ", type2:" + type2); throw e; } return 0; } catch (IOException e) { throw new RuntimeException(e); } }
From source file:com.ebay.erl.mobius.core.model.Tuple.java
License:Apache License
/** * compare two tuples in low level row format. */// w w w. j a v a 2 s . c o m @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { DataInputBuffer d1 = new DataInputBuffer(); d1.reset(b1, s1, l1); DataInputBuffer d2 = new DataInputBuffer(); d2.reset(b2, s2, l2); int _compare_result = Integer.MAX_VALUE; try { // read number of columns from the two tuple int columns_nbr1 = d1.readInt(); int columns_nbr2 = d2.readInt(); int upper_bound = Math.min(columns_nbr1, columns_nbr2); // same column size, start to compare column by column for (int i = 0; i < upper_bound; i++) { byte type1 = d1.readByte(); byte type2 = d2.readByte(); _COLUMN_COMPARATOR.setType(type1, type2); _compare_result = _COLUMN_COMPARATOR.compare(d1, d2, this.conf); // comparing for a column has complete if (_compare_result != 0 && _compare_result != Integer.MAX_VALUE) { // has different, return return _compare_result; } } // end of iterating columns until the upper limit // finished all columns comparison(up to the upper-bound), still cannot find difference, // use the column size as the comparing result. _compare_result = columns_nbr1 - columns_nbr2; } catch (IOException e) { throw new RuntimeException(e); } if (_compare_result == Integer.MAX_VALUE) throw new IllegalArgumentException(); return _compare_result; }
From source file:com.ibm.jaql.io.hadoop.SelectSplitInputFormat.java
License:Apache License
@Override public void configure(JobConf conf) { Class<? extends InputFormat> inputFormatCls = conf.getClass(INPUT_FORMAT, null, InputFormat.class); iFormat = ReflectionUtils.newInstance(inputFormatCls, conf); Class<? extends InputSplit> splitCls = conf.getClass(SPLIT_CLASS, null, InputSplit.class); split = ReflectionUtils.newInstance(splitCls, conf); byte[] bytes = ConfUtil.readBinary(conf, SPLIT); DataInputBuffer buffer = new DataInputBuffer(); buffer.reset(bytes, bytes.length);//from w w w. j a v a2s . c om try { split.readFields(buffer); } catch (IOException e) { throw new UndeclaredThrowableException(e); } }
From source file:com.ibm.jaql.json.type.JsonJavaObject.java
License:Apache License
@Override public JsonJavaObject getCopy(JsonValue target) throws Exception { if (target == this) target = null;/* w w w.j av a 2 s . c o m*/ JsonJavaObject t; if (target instanceof JsonJavaObject) { t = (JsonJavaObject) target; } else { t = new JsonJavaObject(); } if (value == null) { t.value = null; } else { if (t.value == null || t.value.getClass() != this.value.getClass()) { t.value = this.value.getClass().newInstance(); } RandomAccessBuffer copyBuffer = new RandomAccessBuffer(); DataOutputStream copyOutput = new DataOutputStream(copyBuffer); this.value.write(copyOutput); copyOutput.flush(); DataInputBuffer copyInput = new DataInputBuffer(); // TODO: cache copyInput.reset(copyBuffer.getBuffer(), 0, copyBuffer.size()); t.value.readFields(copyInput); } return t; }
From source file:com.ibm.jaql.lang.expr.io.FileSplitToRecordFn.java
License:Apache License
@Override public JsonRecord eval(Context context) throws Exception { // { path: string, start: long, length: long, locations: [string...] } if (in == null) { in = new DataInputBuffer(); jpath = new MutableJsonString(); jstart = new MutableJsonLong(); jlength = new MutableJsonLong(); jlocations = new BufferedJsonArray(); values = new JsonValue[] { jpath, jstart, jlength, jlocations }; resultRec = new BufferedJsonRecord(); resultRec.set(NAMES, values, NAMES.length); }/* ww w . j a va 2 s .c om*/ JsonRecord splitRec = (JsonRecord) exprs[0].eval(context); JsonString jsplitClassName = (JsonString) splitRec.get(InputSplitsFn.CLASS_TAG); Class<? extends FileSplit> splitCls = (Class<? extends FileSplit>) ClassLoaderMgr .resolveClass(jsplitClassName.toString()); FileSplit split = (FileSplit) ReflectionUtils.newInstance(splitCls, null); JsonBinary rawSplit = (JsonBinary) splitRec.get(InputSplitsFn.SPLIT_TAG); in.reset(rawSplit.getInternalBytes(), rawSplit.bytesOffset(), rawSplit.bytesLength()); split.readFields(in); JsonArray jlocs = (JsonArray) splitRec.get(InputSplitsFn.LOCATIONS_TAG); jpath.setCopy(split.getPath().toString()); jstart.set(split.getStart()); jlength.set(split.getLength()); if (jlocs != null) { values[3] = jlocs; } else { String[] locs = split.getLocations(); jlocations.resize(locs.length); for (int i = 0; i < locs.length; i++) { jlocations.set(i, new JsonString(locs[i])); } values[3] = jlocations; } return resultRec; }
From source file:com.ibm.jaql.lang.expr.io.ReadSplitFn.java
License:Apache License
@Override public JsonIterator iter(Context context) throws Exception { // Close the previous adapter, if still open: if (adapter != null) { adapter.close();//from w w w . j ava 2 s.c o m adapter = null; } // evaluate the arguments JsonValue args = exprs[0].eval(context); JsonRecord splitRec = (JsonRecord) exprs[1].eval(context); if (splitRec == null) { return JsonIterator.EMPTY; } // get the InputAdapter according to the type HadoopInputAdapter hia = (HadoopInputAdapter) JaqlUtil.getAdapterStore().input.getAdapter(args); adapter = hia; JobConf conf = new JobConf(); // TODO: allow configuration hia.setParallel(conf); // right thing to do? JsonString jsplitClassName = (JsonString) splitRec.get(InputSplitsFn.CLASS_TAG); Class<? extends InputSplit> splitCls = (Class<? extends InputSplit>) ClassLoaderMgr .resolveClass(jsplitClassName.toString()); InputSplit split = (InputSplit) ReflectionUtils.newInstance(splitCls, conf); DataInputBuffer in = new DataInputBuffer(); JsonBinary rawSplit = (JsonBinary) splitRec.get(InputSplitsFn.SPLIT_TAG); in.reset(rawSplit.getInternalBytes(), rawSplit.bytesOffset(), rawSplit.bytesLength()); split.readFields(in); RecordReader<JsonHolder, JsonHolder> rr = hia.getRecordReader(split, conf, Reporter.NULL); return new RecordReaderValueIter(rr); }
From source file:com.mongodb.hadoop.io.BSONWritable.java
License:Apache License
/** * Used by child copy constructors.//from w ww . j av a 2s .c o m */ protected synchronized void copy(final Writable other) { if (other != null) { try { DataOutputBuffer out = new DataOutputBuffer(); other.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); readFields(in); } catch (IOException e) { throw new IllegalArgumentException("map cannot be copied: " + e.getMessage()); } } else { throw new IllegalArgumentException("source map cannot be null"); } }
From source file:com.moz.fiji.mapreduce.lib.graph.NodeCopier.java
License:Apache License
/** * Creates a new node copier.//from ww w. j a va 2 s. c om */ public NodeCopier() { mOutputStream = new ByteArrayOutputStream(); mEncoder = EncoderFactory.get().directBinaryEncoder(mOutputStream, null); mWriter = new SpecificDatumWriter<Node>(Node.SCHEMA$); mDataBuffer = new DataInputBuffer(); mDecoder = DecoderFactory.get().binaryDecoder(mDataBuffer, null); mReader = new SpecificDatumReader<Node>(Node.SCHEMA$); }