Example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataInputBuffer DataInputBuffer.

Prototype

public DataInputBuffer()

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:org.apache.mahout.clustering.dirichlet.TestMapReduce.java

License:Apache License

public void testNormalModelWritableSerialization() throws Exception {
    double[] m = { 1.1, 2.2, 3.3 };
    Model<?> model = new NormalModel(5, new DenseVector(m), 3.3);
    DataOutputBuffer out = new DataOutputBuffer();
    model.write(out);/*from w  w w .  j a va 2s. c om*/
    Model<?> model2 = new NormalModel();
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    model2.readFields(in);
    assertEquals("models", model.toString(), model2.toString());
}

From source file:org.apache.mahout.clustering.dirichlet.TestMapReduce.java

License:Apache License

public void testSampledNormalModelWritableSerialization() throws Exception {
    double[] m = { 1.1, 2.2, 3.3 };
    Model<?> model = new SampledNormalModel(5, new DenseVector(m), 3.3);
    DataOutputBuffer out = new DataOutputBuffer();
    model.write(out);//from ww  w.j av  a 2  s . c o m
    Model<?> model2 = new SampledNormalModel();
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    model2.readFields(in);
    assertEquals("models", model.toString(), model2.toString());
}

From source file:org.apache.mahout.clustering.dirichlet.TestMapReduce.java

License:Apache License

public void testAsymmetricSampledNormalModelWritableSerialization() throws Exception {
    double[] m = { 1.1, 2.2, 3.3 };
    double[] s = { 3.3, 4.4, 5.5 };
    Model<?> model = new AsymmetricSampledNormalModel(5, new DenseVector(m), new DenseVector(s));
    DataOutputBuffer out = new DataOutputBuffer();
    model.write(out);//w  w  w .  j  a v a 2  s  .  c om
    Model<?> model2 = new AsymmetricSampledNormalModel();
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    model2.readFields(in);
    assertEquals("models", model.toString(), model2.toString());
}

From source file:org.apache.mahout.clustering.dirichlet.TestMapReduce.java

License:Apache License

public void testClusterWritableSerialization() throws Exception {
    double[] m = { 1.1, 2.2, 3.3 };
    DirichletCluster cluster = new DirichletCluster(new NormalModel(5, new DenseVector(m), 4), 10);
    DataOutputBuffer out = new DataOutputBuffer();
    cluster.write(out);//from   ww w.  j a  v a 2 s.  c  o m
    DirichletCluster cluster2 = new DirichletCluster();
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    cluster2.readFields(in);
    assertEquals("count", cluster.getTotalCount(), cluster2.getTotalCount());
    assertNotNull("model null", cluster2.getModel());
    assertEquals("model", cluster.getModel().toString(), cluster2.getModel().toString());
}

From source file:org.apache.mahout.hadoop.io.serializer.SerializationTestUtil.java

License:Apache License

/**
 * A utility that tests serialization/deserialization. 
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization//from   w w  w.  j av  a 2 s .c  o  m
 * @param metadata the metadata to pass to the serializer/deserializer
 * @param <K> the class of the item
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static <K> K testSerialization(Configuration conf, K before) throws Exception {

    SerializationFactory factory = new SerializationFactory(conf);
    SerializerBase<K> serializer = (SerializerBase<K>) factory.getSerializer(before.getClass());
    DeserializerBase<K> deserializer = (DeserializerBase<K>) factory.getDeserializer(before.getClass());

    DataOutputBuffer out = new DataOutputBuffer();
    serializer.open(out);
    serializer.serialize(before);
    serializer.close();

    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    deserializer.open(in);
    K after = deserializer.deserialize(null);
    deserializer.close();
    return after;
}

From source file:org.apache.nutch.crawl.MapWritable.java

License:Apache License

/**
 * Copy constructor. This constructor makes a deep copy, using serialization /
 * deserialization to break any possible references to contained objects.
 * //from   ww w.  jav a 2  s .c o  m
 * @param map map to copy from
 */
public MapWritable(MapWritable map) {
    if (map != null) {
        try {
            DataOutputBuffer dob = new DataOutputBuffer();
            map.write(dob);
            DataInputBuffer dib = new DataInputBuffer();
            dib.reset(dob.getData(), dob.getLength());
            readFields(dib);
        } catch (IOException e) {
            throw new IllegalArgumentException(
                    "this map cannot be copied: " + StringUtils.stringifyException(e));
        }
    }
}

From source file:org.apache.nutch.crawl.TestMapWritable.java

License:Apache License

/** Utility method for testing writables, from hadoop code */
public void testWritable(Writable before) throws Exception {
    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);/*from w w w  .j  a v a2  s .  c om*/

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    Writable after = (Writable) before.getClass().newInstance();
    after.readFields(dib);

    assertEquals(before, after);
}

From source file:org.apache.nutch.crawl.TestMapWritable.java

License:Apache License

public void testRecycling() throws Exception {
    Text value = new Text("value");
    Text key1 = new Text("a");
    Text key2 = new Text("b");

    MapWritable writable = new MapWritable();
    writable.put(key1, value);/*  w  ww.ja v a 2  s .  c o m*/
    assertEquals(writable.get(key1), value);
    assertNull(writable.get(key2));

    DataOutputBuffer dob = new DataOutputBuffer();
    writable.write(dob);
    writable.clear();
    writable.put(key1, value);
    writable.put(key2, value);
    assertEquals(writable.get(key1), value);
    assertEquals(writable.get(key2), value);

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());
    writable.readFields(dib);
    assertEquals(writable.get(key1), value);
    assertNull(writable.get(key2));
}

From source file:org.apache.nutch.searcher.TestHitDetails.java

License:Apache License

public void testHitDetails() throws Exception {
    final int length = 4;
    final String[] fields = new String[] { "a", "b", "c", "a" };
    final String[] values = new String[] { "foo1", "bar", "baz", "foo2" };

    HitDetails before = new HitDetails(fields, values);

    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);//from  w w  w  .  j  av  a  2s  .  co  m

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    HitDetails after = HitDetails.read(dib);

    assertEquals(length, after.getLength());
    for (int i = 0; i < 3; i++) {
        assertEquals(fields[i], after.getField(i));
        assertEquals(values[i], after.getValue(i));
        assertEquals(values[i], after.getValue(fields[i]));
    }
    String[] vals = after.getValues("a");
    assertEquals(2, vals.length);
    assertEquals("foo1", vals[0]);
    assertEquals("foo2", vals[1]);
    vals = after.getValues("b");
    assertEquals(1, vals.length);
    assertEquals("bar", vals[0]);
    vals = after.getValues("c");
    assertEquals(1, vals.length);
    assertEquals("baz", vals[0]);
}

From source file:org.apache.nutch.util.WritableTestUtils.java

License:Apache License

/** Utility method for testing writables. */
public static Writable writeRead(Writable before, Configuration conf) throws Exception {

    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);//from  w w  w . java2  s  . c o m

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    Writable after = (Writable) before.getClass().newInstance();
    if (conf != null) {
        ((Configurable) after).setConf(conf);
    }
    after.readFields(dib);
    return after;
}