Example usage for org.apache.spark.sql.catalyst.expressions UnsafeRow getSizeInBytes

Introduction

In this page you can find the example usage for org.apache.spark.sql.catalyst.expressions UnsafeRow getSizeInBytes.

Prototype

public int getSizeInBytes() 

Source Link

Usage

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.map.UnsafeFixedWidthMonotonicAggregationMap.java

License:Apache License

public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow unsafeGroupingKeyRow) {
    // Probe our map using the serialized key
    final edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.Location loc = map.lookup(
            unsafeGroupingKeyRow.getBaseObject(), unsafeGroupingKeyRow.getBaseOffset(),
            unsafeGroupingKeyRow.getSizeInBytes());
    if (!loc.isDefined()) {
        // This is the first time that we've seen this grouping key, so we'll insert a copy of the
        // empty aggregation buffer into the map:
        boolean putSucceeded = loc.putNewKey(unsafeGroupingKeyRow.getBaseObject(),
                unsafeGroupingKeyRow.getBaseOffset(), unsafeGroupingKeyRow.getSizeInBytes(),
                emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length);
        if (!putSucceeded) {
            return null;
        }// w w w . j  a v a2s .c  o m
    }

    // Reset the pointer to point to the value that we just stored or looked up:
    final MemoryLocation address = loc.getValueAddress();
    currentAggregationBuffer.pointTo(address.getBaseObject(), address.getBaseOffset(),
            aggregationBufferSchema.length(), loc.getValueLength());
    return currentAggregationBuffer;
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.UnsafeFixedWidthSet.java

License:Apache License

public void insert(InternalRow keyRow) {
    UnsafeRow unsafeGroupingKeyRow = (UnsafeRow) keyRow;
    // Probe our set using the serialized key
    final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.Location loc = set.lookup(
            unsafeGroupingKeyRow.getBaseObject(), unsafeGroupingKeyRow.getBaseOffset(),
            unsafeGroupingKeyRow.getSizeInBytes());

    if (!loc.isDefined()) {
        // This is the first time that we've seen this grouping key, so we'll insert a copy of the
        // empty aggregation buffer into the set:
        boolean putSucceeded = loc.putNewKey(unsafeGroupingKeyRow.getBaseObject(),
                unsafeGroupingKeyRow.getBaseOffset(), unsafeGroupingKeyRow.getSizeInBytes());

        if (!putSucceeded)
            throw new RuntimeException(
                    "Not enough memory to copy value " + unsafeGroupingKeyRow.toString() + " into set.");
        //System.out.println("Inserted " + unsafeGroupingKeyRow + " into set. # entries: " + set.numElements());
    }/*w ww. ja  v a 2  s . c o  m*/
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.UnsafeFixedWidthSet.java

License:Apache License

public void ifNotExistsInsert(InternalRow keyRow, HashSet diffSet) {
    UnsafeRow unsafeGroupingKeyRow = (UnsafeRow) keyRow;
    // Probe our set using the serialized key
    final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.Location loc = set.lookup(
            unsafeGroupingKeyRow.getBaseObject(), unsafeGroupingKeyRow.getBaseOffset(),
            unsafeGroupingKeyRow.getSizeInBytes());
    if (!loc.isDefined())
        diffSet.insert(keyRow);// w  w w . j a v a2  s  .c  om
}

---