Example usage for org.apache.hadoop.io ArrayWritable getValueClass

Introduction

In this page you can find the example usage for org.apache.hadoop.io ArrayWritable getValueClass.

Prototype

public Class getValueClass() 

Source Link

Usage

From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java

License:Apache License

public ArrayWritable readVector(ArrayWritable aw) throws IOException {
    if (aw == null) {
        aw = new ArrayWritable(Writable.class);
    } /* else if (!aw.getValueClass().equals(clazz)) {
           throw new RuntimeException("value class has to be "+clazz.getCanonicalName());
         }*//*from   ww w  .  jav a  2  s. c  om*/
    int length = in.readVectorHeader();
    Writable[] writables = new Writable[length];
    for (int i = 0; i < length; i++) {
        Writable w = null;
        if (aw.getValueClass() == TypedBytesWritable.class) {
            w = readRaw();
        } else {
            w = this.read();
            if (w instanceof NullWritable) {
                w = null;
            }
        }
        writables[i] = w;
    }

    aw.set(writables);
    return aw;
}

---

From source file:com.ricemap.spateDB.mapred.SpatialRecordReader.java

License:Apache License

/**
 * Reads all shapes left in the current block in one shot.
 * @param shapes//from ww w.  j  a  va2  s.  co m
 * @return
 * @throws IOException
 */
protected boolean nextShapes(ArrayWritable shapes) throws IOException {
    // Prepare a vector that will hold all objects in this 
    Vector<Shape> vshapes = new Vector<Shape>();
    try {
        Shape stockObject = (Shape) shapes.getValueClass().newInstance();
        // Reached the end of this split
        if (getPos() >= end)
            return false;

        long initialReadPos = getPos();
        long readBytes = 0;

        // Read all shapes in this block
        while (vshapes.size() < maxShapesInOneRead && readBytes < maxBytesInOneRead && nextShape(stockObject)) {
            vshapes.add(stockObject.clone());
            readBytes = getPos() - initialReadPos;
        }

        // Store them in the return value
        shapes.set(vshapes.toArray(new Shape[vshapes.size()]));

        return vshapes.size() > 0;
    } catch (InstantiationException e1) {
        e1.printStackTrace();
    } catch (IllegalAccessException e1) {
        e1.printStackTrace();
    } catch (OutOfMemoryError e) {
        LOG.error("Error reading shapes. Stopped with " + vshapes.size() + " shapes");
        throw e;
    }
    return false;
}

---

From source file:com.uber.hoodie.hadoop.SafeParquetRecordReaderWrapper.java

License:Apache License

public SafeParquetRecordReaderWrapper(RecordReader<NullWritable, ArrayWritable> parquetReader) {
    this.parquetReader = parquetReader;
    ArrayWritable arrayWritable = parquetReader.createValue();
    this.valueClass = arrayWritable.getValueClass();
    this.numValueFields = arrayWritable.get().length;
}

---

From source file:edu.umn.cs.spatialHadoop.mapred.SpatialRecordReader.java

License:Open Source License

/**
 * Reads all shapes left in the current block in one shot. This function
 * runs a loop where it keeps reading shapes by calling the method
 * {@link #nextShape(Shape)} until one of the following conditions happen.
 * 1. The whole file is read. No more records to read.
 * 2. Number of parsed records reaches the threshold defined by the
 *    configuration parameter spatialHadoop.mapred.MaxShapesPerRead.
 *    To disable this check, set the configuration parameter to -1
 * 3. Total size of parsed data from file reaches the threshold defined by
 *    the configuration parameter spatialHadoop.mapred.MaxBytesPerRead.
 *    To disable this check, set the configuration parameter to -1.
 * /*from  w  ww.  ja  v  a  2s .  c  o m*/
 * @param shapes
 * @return
 * @throws IOException
 */
protected boolean nextShapes(ArrayWritable shapes) throws IOException {
    // Prepare a vector that will hold all objects in this 
    Vector<Shape> vshapes = new Vector<Shape>();
    try {
        Shape stockObject = (Shape) shapes.getValueClass().newInstance();
        // Reached the end of this split
        if (getFilePosition() >= end)
            return false;

        long initialReadPos = getPos();
        long readBytes = 0;

        // Read all shapes in this block
        while ((maxShapesInOneRead <= 0 || vshapes.size() < maxShapesInOneRead)
                && (maxBytesInOneRead <= 0 || readBytes < maxBytesInOneRead) && nextShape(stockObject)) {
            vshapes.add(stockObject.clone());
            readBytes = getPos() - initialReadPos;
        }

        // Store them in the return value
        shapes.set(vshapes.toArray(new Shape[vshapes.size()]));

        return !vshapes.isEmpty();
    } catch (InstantiationException e1) {
        e1.printStackTrace();
    } catch (IllegalAccessException e1) {
        e1.printStackTrace();
    } catch (OutOfMemoryError e) {
        LOG.error("Error reading shapes. Stopped with " + vshapes.size() + " shapes");
        throw e;
    }
    return false;
}

---