Example usage for org.apache.spark.unsafe.memory MemoryLocation getBaseOffset

Introduction

In this page you can find the example usage for org.apache.spark.unsafe.memory MemoryLocation getBaseOffset.

Prototype

public final long getBaseOffset() 

Source Link

Usage

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.java

License:Apache License

/**
 * Looks up a key, and saves the result in provided `loc`.
 *
 * This is a thread-safe version of `lookup`, could be used by multiple threads.
 *///w w w . j  a va  2  s .com
public void safeLookup(Object keyBase, long keyOffset, int keyLength,
        edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.Location loc) {
    assert (longArray != null);

    if (enablePerfMetrics) {
        numKeyLookups++;
    }
    final int hashcode = HASHER.hashUnsafeWords(keyBase, keyOffset, keyLength);
    int pos = hashcode & mask;
    int step = 1;
    while (true) {
        if (enablePerfMetrics) {
            numProbes++;
        }
        if (longArray.get(pos * 2) == 0) {
            // This is a new key.
            loc.with(pos, hashcode, false);
            return;
        } else {
            long stored = longArray.get(pos * 2 + 1);
            if ((int) (stored) == hashcode) {
                // Full hash code matches.  Let's compare the keys for equality.
                loc.with(pos, hashcode, true);
                if (loc.getKeyLength() == keyLength) {
                    final MemoryLocation keyAddress = loc.getKeyAddress();
                    final Object storedkeyBase = keyAddress.getBaseObject();
                    final long storedkeyOffset = keyAddress.getBaseOffset();
                    final boolean areEqual = ByteArrayMethods.arrayEquals(keyBase, keyOffset, storedkeyBase,
                            storedkeyOffset, keyLength);
                    if (areEqual) {
                        return;
                    } else {
                        if (enablePerfMetrics) {
                            numHashCollisions++;
                        }
                    }
                }
            }
        }
        pos = (pos + step) & mask;
        step++;
    }
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.map.UnsafeFixedWidthMonotonicAggregationMap.java

License:Apache License

public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow unsafeGroupingKeyRow) {
    // Probe our map using the serialized key
    final edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.Location loc = map.lookup(
            unsafeGroupingKeyRow.getBaseObject(), unsafeGroupingKeyRow.getBaseOffset(),
            unsafeGroupingKeyRow.getSizeInBytes());
    if (!loc.isDefined()) {
        // This is the first time that we've seen this grouping key, so we'll insert a copy of the
        // empty aggregation buffer into the map:
        boolean putSucceeded = loc.putNewKey(unsafeGroupingKeyRow.getBaseObject(),
                unsafeGroupingKeyRow.getBaseOffset(), unsafeGroupingKeyRow.getSizeInBytes(),
                emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length);
        if (!putSucceeded) {
            return null;
        }//from  w w w  .j  a v  a 2 s .  co  m
    }

    // Reset the pointer to point to the value that we just stored or looked up:
    final MemoryLocation address = loc.getValueAddress();
    currentAggregationBuffer.pointTo(address.getBaseObject(), address.getBaseOffset(),
            aggregationBufferSchema.length(), loc.getValueLength());
    return currentAggregationBuffer;
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.map.UnsafeFixedWidthMonotonicAggregationMap.java

License:Apache License

/**
 * Returns an iterator over the keys and values in this map. This uses destructive iterator of
 * BytesToBytesMap. So it is illegal to call any other method on this map after `iterator()` has
 * been called.//www  . j  av  a 2s .  com
 * <p>
 * For efficiency, each call returns the same object.
 */
public KVIterator<UnsafeRow, UnsafeRow> iterator() {
    return new KVIterator<UnsafeRow, UnsafeRow>() {

        private final edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.MapIterator mapLocationIterator = map
                .iterator();
        private final UnsafeRow key = new UnsafeRow();
        private final UnsafeRow value = new UnsafeRow();

        @Override
        public boolean next() {
            if (mapLocationIterator.hasNext()) {
                final edu.ucla.cs.wis.bigdatalog.spark.storage.map.BytesToBytesMap.Location loc = mapLocationIterator
                        .next();
                final MemoryLocation keyAddress = loc.getKeyAddress();
                final MemoryLocation valueAddress = loc.getValueAddress();
                key.pointTo(keyAddress.getBaseObject(), keyAddress.getBaseOffset(), groupingKeySchema.length(),
                        loc.getKeyLength());
                value.pointTo(valueAddress.getBaseObject(), valueAddress.getBaseOffset(),
                        aggregationBufferSchema.length(), loc.getValueLength());
                return true;
            } else {
                return false;
            }
        }

        @Override
        public UnsafeRow getKey() {
            return key;
        }

        @Override
        public UnsafeRow getValue() {
            return value;
        }

        @Override
        public void close() {
            // Do nothing.
        }
    };
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.map.UnsafeFixedWidthMonotonicAggregationMap.java

License:Apache License

private void write(byte[] buffer, MemoryLocation addr, int length, ObjectOutput out)
        throws java.io.IOException {
    if (buffer.length < length)
        buffer = new byte[length];

    Platform.copyMemory(addr.getBaseObject(), addr.getBaseOffset(), buffer, Platform.BYTE_ARRAY_OFFSET, length);
    out.write(buffer, 0, length);/*from   w ww  .  j ava  2s  .  c o  m*/
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.java

License:Apache License

/**
 * Looks up a key, and saves the result in provided `loc`.
 *
 * This is a thread-safe version of `lookup`, could be used by multiple threads.
 *//*ww  w  .  j  a va  2  s .  co m*/
public void safeLookup(Object keyBase, long keyOffset, int keyLength, BytesSet.Location loc) {
    assert (longArray != null);

    if (enablePerfMetrics) {
        numKeyLookups++;
    }
    final int hashcode = HASHER.hashUnsafeWords(keyBase, keyOffset, keyLength);
    int pos = hashcode & mask;
    int step = 1;
    while (true) {
        if (enablePerfMetrics) {
            numProbes++;
        }
        if (longArray.get(pos * 2) == 0) {
            // This is a new key.
            loc.with(pos, hashcode, false);
            return;
        } else {
            long stored = longArray.get(pos * 2 + 1);
            if ((int) (stored) == hashcode) {
                // Full hash code matches.  Let's compare the keys for equality.
                loc.with(pos, hashcode, true);
                if (loc.getKeyLength() == keyLength) {
                    final MemoryLocation keyAddress = loc.getKeyAddress();
                    final Object storedkeyBase = keyAddress.getBaseObject();
                    final long storedkeyOffset = keyAddress.getBaseOffset();
                    final boolean areEqual = ByteArrayMethods.arrayEquals(keyBase, keyOffset, storedkeyBase,
                            storedkeyOffset, keyLength);
                    if (areEqual) {
                        return;
                    } else {
                        if (enablePerfMetrics) {
                            numHashCollisions++;
                        }
                    }
                }
            }
        }
        pos = (pos + step) & mask;
        step++;
    }
}

---

From source file:edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.UnsafeFixedWidthSet.java

License:Apache License

/**
 * Return the aggregation buffer for the current group. For efficiency, all calls to this method
 * return the same object. If additional memory could not be allocated, then this method will
 * signal an error by returning null./*  w ww .  j a  va2s .  c o m*/
 */
/*public UnsafeRow getAggregationBuffer(InternalRow groupingKey) {
final UnsafeRow unsafeGroupingKeyRow = this.groupingKeyProjection.apply(groupingKey);
        
return getAggregationBufferFromUnsafeRow(unsafeGroupingKeyRow);
}*/

/*public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow unsafeGroupingKeyRow) {
// Probe our set using the serialized key
final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.Location loc = set.lookup(
        unsafeGroupingKeyRow.getBaseObject(),
        unsafeGroupingKeyRow.getBaseOffset(),
        unsafeGroupingKeyRow.getSizeInBytes());
if (!loc.isDefined()) {
    // This is the first time that we've seen this grouping key, so we'll insert a copy of the
    // empty aggregation buffer into the set:
    boolean putSucceeded = loc.putNewKey(
            unsafeGroupingKeyRow.getBaseObject(),
            unsafeGroupingKeyRow.getBaseOffset(),
            unsafeGroupingKeyRow.getSizeInBytes()//,
            //emptyAggregationBuffer,
            //Platform.BYTE_ARRAY_OFFSET,
            //emptyAggregationBuffer.length
    );
    if (!putSucceeded) {
        return null;
    }
}
        
// Reset the pointer to point to the value that we just stored or looked up:
final MemoryLocation address = loc.getValueAddress();
currentAggregationBuffer.pointTo(
        address.getBaseObject(),
        address.getBaseOffset(),
        aggregationBufferSchema.length(),
        loc.getValueLength()
);
return currentAggregationBuffer;
}/*
        
/**
 * Returns an iterator over the keys and values in this set. This uses destructive iterator of
 * BytesToBytesMap. So it is illegal to call any other method on this set after `iterator()` has
 * been called.
 *
 * For efficiency, each call returns the same object.
 */
public Iterator<InternalRow> iterator() {
    return new Iterator<InternalRow>() {

        private final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.SetIterator setLocationIterator = set
                .iterator();
        private final UnsafeRow key = new UnsafeRow();
        //private final UnsafeRow value = new UnsafeRow();

        @Override
        public boolean hasNext() {
            return setLocationIterator.hasNext();
        }
        /*    if (setLocationIterator.hasNext()) {
            final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.Location loc = setLocationIterator.next();
            final MemoryLocation keyAddress = loc.getKeyAddress();
            //final MemoryLocation valueAddress = loc.getValueAddress();
            key.pointTo(
                    keyAddress.getBaseObject(),
                    keyAddress.getBaseOffset(),
                    groupingKeySchema.length(),
                    loc.getKeyLength()
            );
            System.out.println("next: " + key.toString());
            /*value.pointTo(
                    valueAddress.getBaseObject(),
                    valueAddress.getBaseOffset(),
                    aggregationBufferSchema.length(),
                    loc.getValueLength()
            );*
            return true;
        } else {
            return false;
        }
        }*/

        @Override
        public UnsafeRow next() {
            final edu.ucla.cs.wis.bigdatalog.spark.storage.set.hashset.BytesSet.Location loc = setLocationIterator
                    .next();
            final MemoryLocation keyAddress = loc.getKeyAddress();
            key.pointTo(keyAddress.getBaseObject(), keyAddress.getBaseOffset(), groupingKeySchema.length(),
                    loc.getKeyLength());

            return key;
        }

        /*@Override
        public UnsafeRow getValue() {
        return value;
        }*/

        @Override
        public void remove() {
            setLocationIterator.remove();
        }

        /*@Override
        public void close() {
        // Do nothing.
        }*/
    };
}

---