List of usage examples for org.apache.commons.lang3.tuple MutablePair getRight
@Override
public R getRight()
From source file:org.apache.apex.malhar.lib.window.accumulation.Average.java
@Override public MutablePair<Double, Long> accumulate(MutablePair<Double, Long> accu, Double input) { accu.setLeft(accu.getLeft() * ((double) accu.getRight() / (accu.getRight() + 1)) + input / (accu.getRight() + 1)); accu.setRight(accu.getRight() + 1);/*from www.ja v a 2 s .c o m*/ return accu; }
From source file:org.apache.apex.malhar.lib.window.accumulation.Average.java
@Override public MutablePair<Double, Long> merge(MutablePair<Double, Long> accu1, MutablePair<Double, Long> accu2) { accu1.setLeft(accu1.getLeft() * ((double) accu1.getRight() / accu1.getRight() + accu2.getRight()) + accu2.getLeft() * ((double) accu2.getRight() / accu1.getRight() + accu2.getRight())); accu1.setRight(accu1.getRight() + accu2.getRight()); return accu1; }
From source file:org.apache.giraph.comm.flow_control.CreditBasedFlowControl.java
@Override public void messageAckReceived(int taskId, long requestId, int response) { boolean ignoreCredit = shouldIgnoreCredit(response); short credit = getCredit(response); int timestamp = getTimestamp(response); MutablePair<AdjustableSemaphore, Integer> pair = (MutablePair<AdjustableSemaphore, Integer>) perWorkerOpenRequestMap .get(taskId);//from ww w .ja v a 2s. c om AdjustableSemaphore openRequestPermit = pair.getLeft(); // Release a permit on open requests if we received ACK of a request other // than a Resume request (resume requests are always sent regardless of // number of open requests) if (!resumeRequestsId.get(taskId).remove(requestId)) { openRequestPermit.release(); } else if (LOG.isDebugEnabled()) { LOG.debug("messageAckReceived: ACK of resume received from " + taskId + " timestamp=" + timestamp); } if (!ignoreCredit) { synchronized (pair) { if (compareTimestamps(timestamp, pair.getRight()) > 0) { pair.setRight(timestamp); openRequestPermit.setMaxPermits(credit); } else if (LOG.isDebugEnabled()) { LOG.debug("messageAckReceived: received out-of-order messages." + "Received timestamp=" + timestamp + " and current timestamp=" + pair.getRight()); } } } // Since we received a response and we changed the credit of the sender // client, we may be able to send some more requests to the sender // client. So, we try to send as much request as we can to the sender // client. trySendCachedRequests(taskId); }
From source file:org.apache.giraph.comm.flow_control.CreditBasedFlowControl.java
/** * Process a resume signal came from a given worker * * @param clientId id of the worker that sent the signal * @param credit the credit value sent along with the resume signal * @param requestId timestamp (request id) of the resume signal */// w ww.j a v a2s . c o m public void processResumeSignal(int clientId, short credit, long requestId) { int timestamp = (int) (requestId & 0xFFFF); if (LOG.isDebugEnabled()) { LOG.debug("processResumeSignal: resume signal from " + clientId + " with timestamp=" + timestamp); } MutablePair<AdjustableSemaphore, Integer> pair = (MutablePair<AdjustableSemaphore, Integer>) perWorkerOpenRequestMap .get(clientId); synchronized (pair) { if (compareTimestamps(timestamp, pair.getRight()) > 0) { pair.setRight(timestamp); pair.getLeft().setMaxPermits(credit); } else if (LOG.isDebugEnabled()) { LOG.debug("processResumeSignal: received out-of-order messages. " + "Received timestamp=" + timestamp + " and current timestamp=" + pair.getRight()); } } trySendCachedRequests(clientId); }
From source file:org.apache.giraph.ooc.data.DiskBackedDataStore.java
/** * Adds a data entry for a given partition to the current data store. If data * of a given partition in data store is already offloaded to disk, adds the * data entry to appropriate raw data buffer list. * * @param partitionId id of the partition to add the data entry to * @param entry data entry to add// w ww. j a v a2 s. co m */ protected void addEntry(int partitionId, T entry) { // Addition of data entries to a data store is much more common than // out-of-core operations. Besides, in-memory data store implementations // existing in the code base already account for parallel addition to data // stores. Therefore, using read lock would optimize for parallel addition // to data stores, specially for cases where the addition should happen for // partitions that are entirely in memory. ReadWriteLock rwLock = getPartitionLock(partitionId); rwLock.readLock().lock(); if (hasPartitionDataOnDisk.contains(partitionId)) { List<T> entryList = new ArrayList<>(); entryList.add(entry); int entrySize = entrySerializedSize(entry); MutablePair<Integer, List<T>> newPair = new MutablePair<>(entrySize, entryList); Pair<Integer, List<T>> oldPair = dataBuffers.putIfAbsent(partitionId, newPair); if (oldPair != null) { synchronized (oldPair) { newPair = (MutablePair<Integer, List<T>>) oldPair; newPair.setLeft(oldPair.getLeft() + entrySize); newPair.getRight().add(entry); } } } else { addEntryToInMemoryPartitionData(partitionId, entry); } rwLock.readLock().unlock(); }
From source file:org.apache.giraph.ooc.data.OutOfCoreDataManager.java
/** * Adds a data entry for a given partition to the current data store. If data * of a given partition in data store is already offloaded to disk, adds the * data entry to appropriate raw data buffer list. * * @param partitionId id of the partition to add the data entry to * @param entry data entry to add/*from w w w.jav a2s. c om*/ */ protected void addEntry(int partitionId, T entry) { // Addition of data entries to a data store is much more common than // out-of-core operations. Besides, in-memory data store implementations // existing in the code base already account for parallel addition to data // stores. Therefore, using read lock would optimize for parallel addition // to data stores, specially for cases where the addition should happen for // partitions that are entirely in memory. ReadWriteLock rwLock = getPartitionLock(partitionId); rwLock.readLock().lock(); if (hasPartitionDataOnDisk.contains(partitionId)) { List<T> entryList = new ArrayList<>(); entryList.add(entry); int entrySize = entrySerializedSize(entry); MutablePair<Integer, List<T>> newPair = new MutablePair<>(entrySize, entryList); Pair<Integer, List<T>> oldPair = dataBuffers.putIfAbsent(partitionId, newPair); if (oldPair != null) { synchronized (oldPair) { newPair = (MutablePair<Integer, List<T>>) oldPair; newPair.setLeft(oldPair.getLeft() + entrySize); newPair.getRight().add(entry); } } } else { addEntryToImMemoryPartitionData(partitionId, entry); } rwLock.readLock().unlock(); }
From source file:org.apache.giraph.ooc.DiskBackedPartitionStore.java
@Override public <M extends Writable> void addPartitionIncomingMessages(int partitionId, VertexIdMessages<I, M> messages) throws IOException { if (conf.getIncomingMessageClasses().useMessageCombiner()) { ((MessageStore<I, M>) incomingMessageStore).addPartitionMessages(partitionId, messages); } else {/* w w w. jav a 2 s .com*/ MetaPartition meta = partitions.get(partitionId); checkNotNull(meta, "addPartitionIncomingMessages: trying to add " + "messages to partition " + partitionId + " which does not exist " + "in the partition set of this worker!"); synchronized (meta) { switch (meta.getState()) { case INACTIVE: case ACTIVE: // A partition might be in memory, but its message store might still // be on disk. This happens because while we are loading the partition // to memory, we only load its current messages, not the incoming // messages. If a new superstep has been started, while the partition // is still in memory, the incoming message store in the previous // superstep (which is the current messages in the current superstep) // is on disk. // This may also happen when a partition is offloaded to disk while // it was unprocessed, and then again loaded in the same superstep for // processing. Boolean isMsgOnDisk = incomingMessagesOnDisk.get(partitionId); if (isMsgOnDisk == null || !isMsgOnDisk) { ((MessageStore<I, M>) incomingMessageStore).addPartitionMessages(partitionId, messages); break; } // Continue to IN_TRANSIT and ON_DISK cases as the partition is in // memory, but it's messages are not yet loaded // CHECKSTYLE: stop FallThrough case IN_TRANSIT: case ON_DISK: // CHECKSTYLE: resume FallThrough List<VertexIdMessages<I, Writable>> newMessages = new ArrayList<VertexIdMessages<I, Writable>>(); newMessages.add((VertexIdMessages<I, Writable>) messages); int length = messages.getSerializedSize(); Pair<Integer, List<VertexIdMessages<I, Writable>>> newPair = new MutablePair<>(length, newMessages); messageBufferRWLock.readLock().lock(); Pair<Integer, List<VertexIdMessages<I, Writable>>> oldPair = pendingIncomingMessages .putIfAbsent(partitionId, newPair); if (oldPair != null) { synchronized (oldPair) { MutablePair<Integer, List<VertexIdMessages<I, Writable>>> pair = (MutablePair<Integer, List<VertexIdMessages<I, Writable>>>) oldPair; pair.setLeft(pair.getLeft() + length); pair.getRight().add((VertexIdMessages<I, Writable>) messages); } } messageBufferRWLock.readLock().unlock(); // In the case that the number of partitions is asked to be fixed by // the user, we should offload the message buffers as necessary. if (isNumPartitionsFixed && pendingIncomingMessages.get(partitionId).getLeft() > minBuffSize) { try { spillPartitionMessages(partitionId); } catch (IOException e) { throw new IllegalStateException("addPartitionIncomingMessages: " + "spilling message buffers for partition " + partitionId + " failed!"); } } break; default: throw new IllegalStateException("addPartitionIncomingMessages: " + "illegal state " + meta.getState() + " for partition " + meta.getId()); } } } }
From source file:org.apache.giraph.ooc.DiskBackedPartitionStore.java
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings("SF_SWITCH_FALLTHROUGH") public void addPartitionEdges(Integer partitionId, VertexIdEdges<I, E> edges) { if (!isInitialized.get()) { initialize();// www . j a v a2s .c o m } MetaPartition meta = new MetaPartition(partitionId); MetaPartition temp = partitions.putIfAbsent(partitionId, meta); if (temp != null) { meta = temp; } boolean createPartition = false; synchronized (meta) { switch (meta.getState()) { case INIT: Partition<I, V, E> partition = conf.createPartition(partitionId, context); meta.setPartition(partition); // This is set to processed so that in the very next iteration cycle, // when startIteration is called, all partitions seem to be processed // and ready for the next iteration cycle. Otherwise, startIteration // fails in its sanity check due to finding an unprocessed partition. meta.setProcessed(true); numPartitionsInMem.getAndIncrement(); meta.setState(State.INACTIVE); synchronized (processedPartitions) { processedPartitions.get(State.INACTIVE).add(partitionId); processedPartitions.notifyAll(); } createPartition = true; // Continue to INACTIVE case to add the edges to the partition // CHECKSTYLE: stop FallThrough case INACTIVE: // CHECKSTYLE: resume FallThrough edgeStore.addPartitionEdges(partitionId, edges); break; case IN_TRANSIT: case ON_DISK: // Adding edges to in-memory buffer of the partition List<VertexIdEdges<I, E>> newEdges = new ArrayList<VertexIdEdges<I, E>>(); newEdges.add(edges); int length = edges.getSerializedSize(); Pair<Integer, List<VertexIdEdges<I, E>>> newPair = new MutablePair<>(length, newEdges); edgeBufferRWLock.readLock().lock(); Pair<Integer, List<VertexIdEdges<I, E>>> oldPair = pendingInputEdges.putIfAbsent(partitionId, newPair); if (oldPair != null) { synchronized (oldPair) { MutablePair<Integer, List<VertexIdEdges<I, E>>> pair = (MutablePair<Integer, List<VertexIdEdges<I, E>>>) oldPair; pair.setLeft(pair.getLeft() + length); pair.getRight().add(edges); } } edgeBufferRWLock.readLock().unlock(); // In the case that the number of partitions is asked to be fixed by the // user, we should offload the edge store as necessary. if (isNumPartitionsFixed && pendingInputEdges.get(partitionId).getLeft() > minBuffSize) { try { spillPartitionInputEdgeStore(partitionId); } catch (IOException e) { throw new IllegalStateException("addPartitionEdges: spilling " + "edge store for partition " + partitionId + " failed!"); } } break; default: throw new IllegalStateException( "illegal state " + meta.getState() + " for partition " + meta.getId()); } } // If creation of a new partition is violating the policy of maximum number // of partitions in memory, we should spill a partition to disk. if (createPartition && numPartitionsInMem.get() > maxPartitionsInMem.get()) { swapOnePartitionToDisk(); } }
From source file:org.apache.giraph.ooc.DiskBackedPartitionStore.java
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings("SF_SWITCH_FALLTHROUGH") public void addPartitionVertices(Integer partitionId, ExtendedDataOutput extendedDataOutput) { if (!isInitialized.get()) { initialize();//ww w. j a v a2s. c o m } MetaPartition meta = new MetaPartition(partitionId); MetaPartition temp = partitions.putIfAbsent(partitionId, meta); if (temp != null) { meta = temp; } boolean createPartition = false; synchronized (meta) { switch (meta.getState()) { case INIT: Partition<I, V, E> partition = conf.createPartition(partitionId, context); meta.setPartition(partition); // Look at the comments in 'addPartitionVertices' for why we set the // this to true. meta.setProcessed(true); numPartitionsInMem.getAndIncrement(); meta.setState(State.INACTIVE); synchronized (processedPartitions) { processedPartitions.get(State.INACTIVE).add(partitionId); processedPartitions.notifyAll(); } createPartition = true; // Continue to INACTIVE case to add the vertices to the partition // CHECKSTYLE: stop FallThrough case INACTIVE: // CHECKSTYLE: resume FallThrough meta.getPartition().addPartitionVertices(new VertexIterator<I, V, E>(extendedDataOutput, conf)); break; case IN_TRANSIT: case ON_DISK: // Adding vertices to in-memory buffer of the partition List<ExtendedDataOutput> vertices = new ArrayList<ExtendedDataOutput>(); vertices.add(extendedDataOutput); int length = extendedDataOutput.getPos(); Pair<Integer, List<ExtendedDataOutput>> newPair = new MutablePair<>(length, vertices); vertexBufferRWLock.readLock().lock(); Pair<Integer, List<ExtendedDataOutput>> oldPair = pendingInputVertices.putIfAbsent(partitionId, newPair); if (oldPair != null) { synchronized (oldPair) { MutablePair<Integer, List<ExtendedDataOutput>> pair = (MutablePair<Integer, List<ExtendedDataOutput>>) oldPair; pair.setLeft(pair.getLeft() + length); pair.getRight().add(extendedDataOutput); } } vertexBufferRWLock.readLock().unlock(); // In the case that the number of partitions is asked to be fixed by the // user, we should offload the edge store as necessary. if (isNumPartitionsFixed && pendingInputVertices.get(partitionId).getLeft() > minBuffSize) { try { spillPartitionInputVertexBuffer(partitionId); } catch (IOException e) { throw new IllegalStateException("addPartitionVertices: spilling " + "vertex buffer for partition " + partitionId + " failed!"); } } break; default: throw new IllegalStateException( "illegal state " + meta.getState() + " for partition " + meta.getId()); } } // If creation of a new partition is violating the policy of maximum number // of partitions in memory, we should spill a partition to disk. if (createPartition && numPartitionsInMem.get() > maxPartitionsInMem.get()) { swapOnePartitionToDisk(); } }
From source file:org.apache.hyracks.storage.am.common.TreeIndexTestUtils.java
protected void addFilterField(IIndexTestContext ctx, MutablePair<ITupleReference, ITupleReference> minMax) throws HyracksDataException { //Duplicate the PK field as a filter field at the end of the tuple to be inserted. int filterField = ctx.getFieldCount(); ITupleReference currTuple = ctx.getTuple(); ArrayTupleBuilder filterBuilder = new ArrayTupleBuilder(1); filterBuilder.addField(currTuple.getFieldData(filterField), currTuple.getFieldStart(filterField), currTuple.getFieldLength(filterField)); IBinaryComparator comparator = ctx.getComparatorFactories()[0].createBinaryComparator(); ArrayTupleReference filterOnlyTuple = new ArrayTupleReference(); filterOnlyTuple.reset(filterBuilder.getFieldEndOffsets(), filterBuilder.getByteArray()); if (minMax == null) { minMax = MutablePair.of(filterOnlyTuple, filterOnlyTuple); } else if (compareFilterTuples(minMax.getLeft(), filterOnlyTuple, comparator) > 0) { minMax.setLeft(filterOnlyTuple); } else if (compareFilterTuples(minMax.getRight(), filterOnlyTuple, comparator) < 0) { minMax.setRight(filterOnlyTuple); }/*from w w w . ja v a 2 s . c o m*/ }