Example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer.

Prototype

public DataOutputBuffer() 

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:com.dasasian.chok.lucene.LuceneServer.java

License:Apache License

@Override
public HitsMapWritable search(QueryWritable query, DocumentFrequencyWritable freqs, String[] shards,
        final long timeout, int count, SortWritable sortWritable, FilterWritable filterWritable)
        throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("You are searching with the query: '" + query.getQuery() + "'");
    }//from  ww  w . j av  a2s .c om

    Query luceneQuery = query.getQuery();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Lucene query: " + luceneQuery.toString());
    }

    long completeSearchTime = 0;
    final HitsMapWritable result = new com.dasasian.chok.lucene.HitsMapWritable(getNodeName());
    long start = 0;
    if (LOG.isDebugEnabled()) {
        start = System.currentTimeMillis();
    }
    Sort sort = null;
    if (sortWritable != null) {
        sort = sortWritable.getSort();
    }
    Filter filter = null;
    if (filterWritable != null) {
        filter = filterWritable.getFilter();
    }
    if (filterCache != null && filter != null) {
        CachingWrapperFilter cachedFilter = filterCache.getIfPresent(filter);
        if (cachedFilter == null) {
            cachedFilter = new CachingWrapperFilter(filter);
            filterCache.put(filter, cachedFilter);
        }
        filter = cachedFilter;
    }
    search(luceneQuery, freqs, shards, result, count, sort, timeout, filter);
    if (LOG.isDebugEnabled()) {
        final long end = System.currentTimeMillis();
        LOG.debug("Search took " + (end - start) / 1000.0 + "sec.");
        completeSearchTime += (end - start);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Complete search took " + completeSearchTime / 1000.0 + "sec.");
        final DataOutputBuffer buffer = new DataOutputBuffer();
        result.write(buffer);
        LOG.debug("Result size to transfer: " + buffer.getLength());
    }
    return result;
}

---

From source file:com.datasalt.pangool.BaseTest.java

License:Apache License

protected static void assertSerializable(HadoopSerialization ser, ITuple tuple, boolean debug)
        throws IOException {
    DataInputBuffer input = new DataInputBuffer();
    DataOutputBuffer output = new DataOutputBuffer();
    DatumWrapper<ITuple> wrapper = new DatumWrapper<ITuple>(tuple);
    ser.ser(wrapper, output);//from   ww w. j  a va 2 s  . c  om

    input.reset(output.getData(), 0, output.getLength());
    DatumWrapper<ITuple> wrapper2 = new DatumWrapper<ITuple>();

    wrapper2 = ser.deser(wrapper2, input);
    if (debug) {
        System.out.println("D:" + wrapper2.datum());
    }
    assertEquals(tuple, wrapper2.datum());
}

---

From source file:com.datasalt.pangool.BaseTest.java

License:Apache License

protected static void assertSerializable(TupleSerializer ser, TupleDeserializer deser,
        DatumWrapper<ITuple> tuple, boolean debug) throws IOException {
    DataOutputBuffer output = new DataOutputBuffer();
    ser.open(output);//  w ww.j  a  v a 2s .c  o m
    ser.serialize(tuple);
    ser.close();

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), 0, output.getLength());
    DatumWrapper<ITuple> deserializedTuple = new DatumWrapper<ITuple>();

    deser.open(input);
    deserializedTuple = deser.deserialize(deserializedTuple);
    deser.close();

    if (debug) {
        System.out.println("D:" + deserializedTuple.datum());
    }
    assertEquals(tuple.datum(), deserializedTuple.datum());
}

---

From source file:com.datasalt.pangool.BaseTest.java

License:Apache License

protected void assertSerializable(Tuple tuple, Tuple toReuse, boolean debug) throws IOException {
    HadoopSerialization hadoopSer = new HadoopSerialization(getConf());
    SimpleTupleSerializer ser = new SimpleTupleSerializer(hadoopSer);
    SimpleTupleDeserializer deser = new SimpleTupleDeserializer(tuple.getSchema(), hadoopSer, getConf());

    DataOutputBuffer output = new DataOutputBuffer();
    ser.open(output);//from  ww w  .  j  ava  2 s . c o m
    ser.serialize(tuple);
    ser.close();

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), 0, output.getLength());
    ITuple deserializedTuple = null;

    deser.open(input);
    deserializedTuple = deser.deserialize(null);
    deser.close();

    if (debug) {
        System.out.println("D:" + tuple);
    }
    assertEquals(tuple, deserializedTuple);
}

---

From source file:com.datasalt.pangool.io.TestBitField.java

License:Apache License

static void serDeCheck(BitField bf) throws IOException {
    DataOutputBuffer dob = new DataOutputBuffer();
    DataInputBuffer dib = new DataInputBuffer();
    bf.ser(dob);/*w  ww  .j av a  2  s  .co m*/
    dib.reset(dob.getData(), dob.getData().length);
    // Checking DataInput deser
    BitField newbf = new BitField();
    newbf.set(1000);
    newbf.deser(dib);
    assertEquals(bf.toString() + " vs " + newbf.toString(), 0, bf.compareTo(newbf));
    // Asserting that the rightmost bit is always 0. Important for comparisons
    byte[] arr = newbf.getBackingArray();
    for (int i = 0; i < arr.length; i++) {
        assertEquals(0, arr[i] & 1);
    }

    //Checking byte array deserialization
    newbf = new BitField();
    newbf.set(1000);
    newbf.deser(dob.getData(), 0);
    assertEquals(bf.toString() + " vs " + newbf.toString(), 0, bf.compareTo(newbf));
    // Asserting that the rightmost bit is always 0. Important for comparisons
    arr = newbf.getBackingArray();
    for (int i = 0; i < arr.length; i++) {
        assertEquals(0, arr[i] & 1);
    }
}

---

From source file:com.datasalt.pangool.io.TestBitField.java

License:Apache License

static void checkReturnOfDeser(int bits) throws IOException {
    BitField bf = new BitField();
    bf.set(bits - 1);/*from w  w  w .  j a v a  2s.  c o m*/
    DataOutputBuffer dob = new DataOutputBuffer();
    DataInputBuffer dib = new DataInputBuffer();
    bf.ser(dob);
    dib.reset(dob.getData(), dob.getData().length);
    // Checking DataInput deser
    BitField newbf = new BitField();
    int read = newbf.deser(dib);
    assertEquals("For bits " + bits, ((bits - 1) / 7) + 1, read);

    //Checking byte array deserialization
    newbf = new BitField();
    read = newbf.deser(dob.getData(), 0);
    assertEquals("For bits " + bits, ((bits - 1) / 7) + 1, read);
}

---

From source file:com.datasalt.pangool.tuplemr.mapred.BinaryComparator.java

License:Apache License

@Override
public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();

    try {//from www .j ava2 s .c  om
        ser = new HadoopSerialization(conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datasalt.pangool.tuplemr.mapred.SerializerComparator.java

License:Apache License

@Override
public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();

    try {//  w w  w.  j a v  a2  s  . com
        hadoopSer = new HadoopSerialization(conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datasalt.pangool.tuplemr.mapred.TestComparators.java

License:Apache License

private int compareInBinary1(SortComparator comp, ITuple tuple1, ITuple tuple2) throws IOException {
    DataOutputBuffer buffer1 = new DataOutputBuffer();
    ser.ser(new DatumWrapper(tuple1), buffer1);

    DataOutputBuffer buffer2 = new DataOutputBuffer();
    ser.ser(new DatumWrapper(tuple2), buffer2);

    return comp.compare(buffer1.getData(), 0, buffer1.getLength(), buffer2.getData(), 0, buffer2.getLength());
}

---

From source file:com.datasalt.pangool.tuplemr.serialization.TestSingleFieldDeserializer.java

License:Apache License

@Test
public void testThrift() throws IOException, TupleMRException {
    Configuration conf = getConf();

    ArrayList<Field> fields = new ArrayList<Field>();
    Field field = Field.createObject("a", A.class);
    fields.add(field);/*from  w w  w.  j  a v  a  2 s  .com*/
    Schema schema = new Schema("schema", fields);

    Tuple tuple1 = new Tuple(schema);
    final A a = new A("hola", "colega");
    tuple1.set("a", a);

    final A b = new A("bloblo", "coco");
    Tuple tuple2 = new Tuple(schema);
    tuple2.set("a", b);

    TupleMRConfigBuilder builder = new TupleMRConfigBuilder();
    builder.addIntermediateSchema(schema);
    builder.setGroupByFields("a");
    builder.setOrderBy(new OrderBy().add("a", Order.ASC, new BaseComparator<A>(Type.OBJECT, A.class) {

        @Override
        public int compare(A o1, A o2) {
            assertEquals(a, o1);
            assertEquals(b, o2);
            return 1;
        }

    }));

    TupleMRConfig tupleMRConf = builder.buildConf();
    TupleMRConfig.set(tupleMRConf, conf);

    HadoopSerialization ser = new HadoopSerialization(conf);

    DataOutputBuffer buffer1 = new DataOutputBuffer();
    ser.ser(new DatumWrapper<ITuple>(tuple1), buffer1);

    SingleFieldDeserializer fieldDeser = new SingleFieldDeserializer(conf, tupleMRConf, field.getType(),
            field.getObjectClass());
    A otherA = (A) fieldDeser.deserialize(buffer1.getData(), 0);
    assertEquals(a, otherA);

    DataOutputBuffer buffer2 = new DataOutputBuffer();
    ser.ser(new DatumWrapper<ITuple>(tuple2), buffer2);

    SortComparator sortComparator = new SortComparator();
    sortComparator.setConf(conf);

    System.out.println("buff1: " + Arrays.toString(buffer1.getData()));
    System.out.println("buff2: " + Arrays.toString(buffer2.getData()));

    sortComparator.compare(buffer1.getData(), 0, buffer1.size(), buffer2.getData(), 0, buffer2.size());

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• »