List of usage examples for java.nio LongBuffer position
public final Buffer position(int newPosition)
From source file:edu.cmu.graphchi.shards.QueryShard.java
public EdgeIterator edgeIterator(long fromSrcVertex) { if (!pinIndexToMemory) { final LongBuffer iterBuffer = adjBuffer.duplicate(); final LongBuffer iterPointerBuffer = pointerIdxBuffer.duplicate(); if (fromSrcVertex <= 0) { iterBuffer.position(0); iterPointerBuffer.position(0); } else {//from w ww . j av a2s . c om ShardIndex.IndexEntry entry = null; entry = (pinIndexToMemory ? null : index.lookup(fromSrcVertex)); // ugly PointerPair ptr = findIdxAndPos(fromSrcVertex, entry, iterPointerBuffer, new long[2], true); assert (ptr.cur >= 0); iterBuffer.position((int) ptr.cur); iterPointerBuffer.position(iterPointerBuffer.position() - 1); } return new EdgeIterator() { int idx = iterBuffer.position() - 1; long ptr = (iterPointerBuffer.capacity() > 0 ? iterPointerBuffer.get() : -1); long nextPtr = (iterPointerBuffer.capacity() > 0 ? iterPointerBuffer.get() : -1); long nextOff = VertexIdTranslate.getAux(nextPtr); long curSrc = VertexIdTranslate.getVertexId(ptr); long curDst; byte curType; long vertexPacket; @Override public boolean hasNext() { if (idx < numEdges - 1) { vertexPacket = iterBuffer.get(); if (VertexIdTranslate.isEdgeDeleted(vertexPacket)) { next(); // Skip over deleted edges return hasNext(); } else { return true; } } else { return false; } } @Override public void next() { idx++; if (idx == nextOff) { curSrc = VertexIdTranslate.getVertexId(nextPtr); nextPtr = iterPointerBuffer.get(); nextOff = VertexIdTranslate.getAux(nextPtr); } curDst = VertexIdTranslate.getVertexId(vertexPacket); curType = VertexIdTranslate.getType(vertexPacket); } @Override public long getSrc() { return curSrc; } @Override public long getDst() { return curDst; } @Override public byte getType() { return curType; } @Override public int getIdx() { return idx; } }; } else { if (gammaSeqVertices == null) { return new EdgeIterator() { @Override public boolean hasNext() { return false; } @Override public void next() { } @Override public long getSrc() { return 0; } @Override public long getDst() { return 0; } @Override public byte getType() { return 0; } @Override public int getIdx() { return 0; } }; } long startOff = 0; final LongBuffer iterBuffer = adjBuffer.duplicate(); final Iterator<Long> iterPointerVertices = gammaSeqVertices.iterator(fromSrcVertex); if (fromSrcVertex == 0) { iterBuffer.position(0); } else { int startIdx = gammaSeqVertices.getIndex(fromSrcVertex, true); startOff = gammaSeqOffs.get(startIdx); iterBuffer.position((int) startOff); } final Iterator<Long> iterPointerOffs = gammaSeqOffs.iterator(startOff); return new EdgeIterator() { int idx = iterBuffer.position() - 1; long ptr = (iterPointerVertices.hasNext() ? VertexIdTranslate.encodeVertexPacket((byte) 0, iterPointerVertices.next(), iterPointerOffs.next()) : -1); long nextPtr = (iterPointerVertices.hasNext() ? VertexIdTranslate.encodeVertexPacket((byte) 0, iterPointerVertices.next(), iterPointerOffs.next()) : -1); long nextOff = VertexIdTranslate.getAux(nextPtr); long curSrc = VertexIdTranslate.getVertexId(ptr); long curDst; byte curType; long vertexPacket; @Override public boolean hasNext() { if (idx < numEdges - 1) { vertexPacket = iterBuffer.get(); if (VertexIdTranslate.isEdgeDeleted(vertexPacket)) { next(); // Skip over deleted edges return hasNext(); } else { return true; } } else { return false; } } @Override public void next() { idx++; if (idx == nextOff) { curSrc = VertexIdTranslate.getVertexId(nextPtr); if (iterPointerVertices.hasNext()) { nextPtr = VertexIdTranslate.encodeVertexPacket((byte) 0, iterPointerVertices.next(), iterPointerOffs.next()); } else { // System.out.println("Warning: edgeIterator at " + idx + " but no more left..."); } nextOff = VertexIdTranslate.getAux(nextPtr); } curDst = VertexIdTranslate.getVertexId(vertexPacket); curType = VertexIdTranslate.getType(vertexPacket); } @Override public long getSrc() { return curSrc; } @Override public long getDst() { return curDst; } @Override public byte getType() { return curType; } @Override public int getIdx() { return idx; } }; } }
From source file:edu.cmu.graphchi.shards.QueryShard.java
private PointerPair findIdxAndPos(long vertexId, ShardIndex.IndexEntry sparseIndexEntry, final LongBuffer tmpPtrBuffer, long[] workarr, boolean nextGreatest) { if (tmpPtrBuffer != null) { assert (sparseIndexEntry.vertex <= vertexId); // Not pinned if (tmpPtrBuffer.capacity() == 0) return new PointerPair(-1, -1); int vertexSeq = sparseIndexEntry.vertexSeq; long curvid = sparseIndexEntry.vertex; if (sparseIndexEntry.nextEntry == null || nextGreatest) { // Linear search tmpPtrBuffer.position(vertexSeq); long ptr = tmpPtrBuffer.get(); while (curvid <= vertexId) { try { curvid = VertexIdTranslate.getVertexId(ptr); if (curvid == vertexId || (curvid >= vertexId && nextGreatest)) { return new PointerPair(VertexIdTranslate.getAux(ptr), VertexIdTranslate.getAux(tmpPtrBuffer.get())); }/* w ww.j ava 2 s. c o m*/ ptr = tmpPtrBuffer.get(); } catch (BufferUnderflowException bufe) { return new PointerPair(-1, -1); } } } else { // Binary search int low = sparseIndexEntry.vertexSeq; int high = sparseIndexEntry.nextEntry.vertexSeq; int n = tmpPtrBuffer.capacity(); while (low <= high) { int idx = ((high + low) / 2); long ptr = tmpPtrBuffer.get(idx); curvid = VertexIdTranslate.getVertexId(ptr); if (curvid == vertexId) { return new PointerPair(VertexIdTranslate.getAux(ptr), VertexIdTranslate.getAux(tmpPtrBuffer.get(idx + 1))); } if (curvid < vertexId) { low = idx + 1; } else { high = idx - 1; } } } return new PointerPair(-1, -1); } else { if (gammaSeqVertices == null) { // Empty shard return new PointerPair(-1, -1); } // Pinned int idx = gammaSeqVertices.getIndex(vertexId); if (idx == -1) return new PointerPair(-1, -1); gammaSeqOffs.getTwo(idx, workarr); return new PointerPair(workarr[0], workarr[1]); } }
From source file:edu.cmu.graphchi.shards.QueryShard.java
public Long find(byte edgeType, long src, long dst) { final LongBuffer tmpAdjBuffer = adjBuffer.duplicate(); final LongBuffer tmpPointerBuffer = (pinIndexToMemory ? null : pointerIdxBuffer.duplicate()); ShardIndex.IndexEntry indexEntry = (pinIndexToMemory ? null : index.lookup(src)); PointerPair ptr = findIdxAndPos(src, indexEntry, tmpPointerBuffer, new long[2]); if (ptr.cur != (-1L)) { long nextPtr = ptr.next; int n = (int) (nextPtr - ptr.cur); long adjOffset = ptr.cur; if (n < 32) { // linear search tmpAdjBuffer.position((int) adjOffset); for (int i = 0; i < n; i++) { long e = tmpAdjBuffer.get(); long v = VertexIdTranslate.getVertexId(e); if (v == dst && VertexIdTranslate.getType(e) == edgeType) { return PointerUtil.encodePointer(shardNum, (int) adjOffset + i); } else if (v > dst) { break; }//from w w w . j a v a2 s .c om } } else { // Binary search int low = (int) adjOffset; int high = low + n; while (low <= high) { int idx = ((high + low) / 2); long e = tmpAdjBuffer.get(idx); long v = VertexIdTranslate.getVertexId(e); byte type = VertexIdTranslate.getType(e); if (v == dst && type == edgeType) { // Note: edgeType sorting is not accurate as edge type can be changed (by deleting)! return PointerUtil.encodePointer(shardNum, idx); } if (v < dst || (dst == v && type < edgeType)) { low = idx + 1; } else { high = idx - 1; } } } } return null; }
From source file:edu.cmu.graphchi.shards.QueryShard.java
public void queryOut(long vertexId, QueryCallback callback, byte edgeType, boolean ignoreType) { if (isEmpty()) return;//from w ww . j a v a2 s . c o m try { final LongBuffer tmpPointerIdxBuffer = (pointerIdxBuffer != null ? pointerIdxBuffer.duplicate() : null); final LongBuffer tmpAdjBuffer = adjBuffer.duplicate(); boolean immediateReceive = callback.immediateReceive(); ShardIndex.IndexEntry entry = null; entry = (pinIndexToMemory ? null : index.lookup(vertexId)); // ugly PointerPair ptr = findIdxAndPos(vertexId, entry, tmpPointerIdxBuffer, new long[2]); long curPtr = ptr.cur; if (ptr.cur != (-1L)) { long nextPtr = ptr.next; int n = (int) (nextPtr - curPtr); ArrayList<Long> res = (immediateReceive ? null : new ArrayList<Long>(n)); ArrayList<Long> resPointers = (immediateReceive ? null : new ArrayList<Long>(n)); ArrayList<Byte> resTypes = (immediateReceive ? null : new ArrayList<Byte>(n)); tmpAdjBuffer.position((int) curPtr); for (int i = 0; i < n && tmpAdjBuffer.position() < tmpAdjBuffer.capacity(); i++) { // The latter condition is to fix some bug with the last vertex? long e = tmpAdjBuffer.get(); byte etype = VertexIdTranslate.getType(e); if (etype == edgeType || ignoreType) { if (!immediateReceive) { res.add(VertexIdTranslate.getVertexId(e)); resPointers.add(PointerUtil.encodePointer(shardNum, (int) curPtr + i)); resTypes.add(etype); } else { callback.receiveEdge(vertexId, VertexIdTranslate.getVertexId(e), etype, PointerUtil.encodePointer(shardNum, (int) curPtr + i)); } } } if (!immediateReceive) callback.receiveOutNeighbors(vertexId, res, resTypes, resPointers); } else { if (!immediateReceive) callback.receiveOutNeighbors(vertexId, new ArrayList<Long>(0), new ArrayList<Byte>(0), new ArrayList<Long>(0)); } } catch (FinishQueryException fqe) { // Used for cases when query was early fulfilled throw fqe; } catch (Exception err) { err.printStackTrace(); throw new RuntimeException(err); } }
From source file:edu.cmu.graphchi.shards.QueryShard.java
public void queryOut(List<Long> sortedIds, QueryCallback callback, byte edgeType, boolean ignoreType) { if (isEmpty()) return;// w ww. j av a 2s . co m try { final Timer.Context _timer1 = outEdgePhase1Timer.time(); if (queryCacheSize > 0 && !queryCache.isEmpty()) { if (callback.immediateReceive()) { ArrayList<Long> misses = null; for (long qid : sortedIds) { Long cacheKey = outcachekey(qid, edgeType); long[] cached = (long[]) queryCache.get(cacheKey); if (cached != null) { for (int j = 0; j < cached.length; j++) { long vpacket = cached[j]; callback.receiveEdge(qid, VertexIdTranslate.getVertexId(vpacket), edgeType, PointerUtil.encodePointer(shardNum, (int) VertexIdTranslate.getAux(vpacket))); } } else { if (misses == null) misses = new ArrayList<Long>(); misses.add(qid); } } if (misses != null) cacheMissCounter.inc(misses.size()); if (misses != null) cacheHitCounter.inc(sortedIds.size() - misses.size()); else cacheHitCounter.inc(sortedIds.size()); sortedIds = misses; if (sortedIds == null) return; } else { System.err.println("Caching without immediatereceive not implemented yet"); } } ArrayList<ShardIndex.IndexEntry> indexEntries = new ArrayList<>(sortedIds.size()); if (!pinIndexToMemory) { for (Long a : sortedIds) { indexEntries.add(index.lookup(a)); } } _timer1.stop(); final Timer.Context _timer2 = outEdgePhase2Timer.time(); final LongBuffer tmpPointerIdxBuffer = (pointerIdxBuffer != null ? pointerIdxBuffer.duplicate() : null); final LongBuffer tmpAdjBuffer = adjBuffer.duplicate(); ShardIndex.IndexEntry entry = null; long[] workarr = new long[2]; for (int qIdx = 0; qIdx < sortedIds.size(); qIdx++) { entry = (pinIndexToMemory ? null : indexEntries.get(qIdx)); // ugly long vertexId = sortedIds.get(qIdx); if (qIdx > 0) { if (sortedIds.get(qIdx - 1) >= vertexId) { throw new IllegalArgumentException("Query ids have to be sorted!"); } } PointerPair ptr = findIdxAndPos(vertexId, entry, tmpPointerIdxBuffer, workarr); long curPtr = ptr.cur; if (ptr.cur != (-1L)) { long nextPtr = ptr.next; int n = (int) (nextPtr - curPtr); ArrayList<Long> res = (callback.immediateReceive() ? null : new ArrayList<Long>(n)); ArrayList<Long> resPointers = (callback.immediateReceive() ? null : new ArrayList<Long>(n)); ArrayList<Byte> resTypes = (callback.immediateReceive() ? null : new ArrayList<Byte>(n)); tmpAdjBuffer.position((int) curPtr); long[] cached = (queryCache == null || queryCache.size() >= queryCacheSize || freezeCache ? null : new long[n]); int cachek = 0; for (int i = 0; i < n && tmpAdjBuffer.position() < tmpAdjBuffer.capacity(); i++) { long e = tmpAdjBuffer.get(); byte etype = VertexIdTranslate.getType(e); if (ignoreType || etype == edgeType) { if (!callback.immediateReceive()) { res.add(VertexIdTranslate.getVertexId(e)); resPointers.add(PointerUtil.encodePointer(shardNum, (int) curPtr + i)); resTypes.add(etype); } else { callback.receiveEdge(vertexId, VertexIdTranslate.getVertexId(e), etype, PointerUtil.encodePointer(shardNum, (int) curPtr + i)); if (cached != null) { cached[cachek++] = VertexIdTranslate.encodeVertexPacket(edgeType, VertexIdTranslate.getVertexId(e), curPtr + 1); } } } } if (!callback.immediateReceive()) callback.receiveOutNeighbors(vertexId, res, resTypes, resPointers); if (cached != null) { if (cachek < n) { cached = Arrays.copyOf(cached, cachek); } queryCache.put(outcachekey(vertexId, edgeType), cached); } } else { if (!callback.immediateReceive()) callback.receiveOutNeighbors(vertexId, new ArrayList<Long>(0), new ArrayList<Byte>(0), new ArrayList<Long>(0)); if (queryCache != null && queryCacheSize > queryCache.size() && !freezeCache) { queryCache.put(outcachekey(vertexId, edgeType), new long[0]); } } } _timer2.stop(); } catch (FinishQueryException fqe) { // Used for cases when query was early fulfilled throw fqe; } catch (Exception err) { throw new RuntimeException(err); } }
From source file:edu.cmu.graphchi.shards.QueryShard.java
private void queryIn(Long queryId, QueryCallback callback, byte edgeType, boolean ignoreType) { if (queryCache != null && callback.immediateReceive()) { long[] cached = (long[]) queryCache.get(incacheKey(queryId, edgeType)); if (cached != null && callback.immediateReceive()) { for (int j = 0; j < cached.length; j++) { long ptr = cached[j]; callback.receiveEdge(VertexIdTranslate.getVertexId(ptr), queryId, edgeType, PointerUtil.encodePointer(shardNum, (int) VertexIdTranslate.getAux(ptr))); }/*from w ww . j a v a 2s. c o m*/ // cacheHitCounter.inc(); return; } else { cacheMissCounter.inc(); } } if (queryId < interval.getFirstVertex() || queryId > interval.getLastVertex()) { throw new IllegalArgumentException("Vertex " + queryId + " not part of interval:" + interval); } final LongBuffer tmpBuffer = adjBuffer.duplicate(); try { /* Step 1: collect adj file offsets for the in-edges */ final Timer.Context _timer1 = inEdgePhase1Timer.time(); ArrayList<Integer> offsets = new ArrayList<Integer>(); int END = (1 << 26) - 1; final IntBuffer startBufferTmp = inEdgeStartBuffer.duplicate(); // Binary search to find the start of the vertex int n = inEdgeStartBuffer.capacity() / 2; int low = 0; int high = n - 1; int off = -1; int queryRelative = (int) (queryId - interval.getFirstVertex()); while (low <= high) { int idx = ((high + low) / 2); int v = startBufferTmp.get(idx * 2); if (v == queryRelative) { off = startBufferTmp.get(idx * 2 + 1); break; } if (v < queryRelative) { low = idx + 1; } else { high = idx - 1; } } if (off == (-1)) { if (queryCache != null && queryCacheSize > queryCache.size()) { queryCache.put(incacheKey(queryId, edgeType), new long[0]); } return; } while (off != END) { tmpBuffer.position(off); long edge = tmpBuffer.get(); if (VertexIdTranslate.getVertexId(edge) != queryId) { System.out.println( "Mismatch in edge linkage: " + VertexIdTranslate.getVertexId(edge) + " !=" + queryId); throw new RuntimeException( "Mismatch in edge linkage: " + VertexIdTranslate.getVertexId(edge) + " !=" + queryId); } if (ignoreType || VertexIdTranslate.getType(edge) == edgeType) { offsets.add(off); } off = (int) VertexIdTranslate.getAux(edge); if (off > END) { throw new RuntimeException("Encoding error: " + edge + " --> " + VertexIdTranslate.getVertexId(edge) + " off : " + VertexIdTranslate.getAux(edge)); } if (off != END && (off < 0 || off > tmpBuffer.capacity())) { System.err.println("Illegal off when looking for inedges: " + off + ", capacity:" + tmpBuffer.capacity() + ", shardNum=" + shardNum); } } _timer1.stop(); /* Step 2: collect the vertex ids that contain the offsets by passing over the pointer data */ /* Find beginning */ ArrayList<Long> inNeighbors = (callback.immediateReceive() ? null : new ArrayList<Long>(offsets.size())); ArrayList<Long> inNeighborsPtrs = (callback.immediateReceive() ? null : new ArrayList<Long>(offsets.size())); ArrayList<Byte> edgeTypes = (callback.immediateReceive() ? null : new ArrayList<Byte>(offsets.size())); ArrayList<Long> cached = (queryCache == null || queryCache.size() >= queryCacheSize || freezeCache ? null : new ArrayList<Long>()); final LongBuffer tmpPointerIdxBuffer = (pinIndexToMemory ? null : pointerIdxBuffer.duplicate()); Iterator<Integer> offsetIterator = offsets.iterator(); if (!offsets.isEmpty()) { final Timer.Context _timer2 = inEdgePhase2Timer.time(); int firstOff = offsets.get(0); if (tmpPointerIdxBuffer != null) { final Timer.Context _timer3 = inEdgeIndexLookupTimer.time(); ShardIndex.IndexEntry startIndex = index.lookupByOffset(firstOff * 8); _timer3.stop(); tmpPointerIdxBuffer.position(startIndex.vertexSeq); } int lastOff = firstOff; while (offsetIterator.hasNext()) { off = offsetIterator.next(); if (off - lastOff > 8196 && tmpPointerIdxBuffer != null) { // magic threshold when to consult the index ShardIndex.IndexEntry skipIdx = index.lookupByOffset(off * 8); if (skipIdx.fileOffset > lastOff) { tmpPointerIdxBuffer.position(skipIdx.vertexSeq); } } long vert = findVertexForOff(off, tmpPointerIdxBuffer); if (!callback.immediateReceive()) { inNeighbors.add(vert); inNeighborsPtrs.add(PointerUtil.encodePointer(shardNum, off)); edgeTypes.add(edgeType); // TODO with wild card } else { callback.receiveEdge(vert, queryId, edgeType, PointerUtil.encodePointer(shardNum, off)); } if (cached != null) { cached.add(VertexIdTranslate.encodeVertexPacket(edgeType, vert, off)); } lastOff = off; } _timer2.stop(); if (cached != null) { long[] cachedArr = new long[cached.size()]; for (int j = 0; j < cached.size(); j++) cachedArr[j] = cached.get(j); queryCache.put(incacheKey(queryId, edgeType), cachedArr); } } if (!callback.immediateReceive()) { callback.receiveInNeighbors(queryId, inNeighbors, edgeTypes, inNeighborsPtrs); } } catch (Exception err) { throw new RuntimeException(err); } }