List of usage examples for java.util.concurrent ConcurrentLinkedQueue clear
public void clear()
From source file:com.chinamobile.bcbsp.comm.CombinerTool.java
/** combine the message queues. * @param outgoingQueue//from ww w.ja va2 s . c o m */ private ConcurrentLinkedQueue<IMessage> combine(ConcurrentLinkedQueue<IMessage> outgoingQueue) { // Map of outgoing queues indexed by destination vertex ID. TreeMap<String, ConcurrentLinkedQueue<IMessage>> outgoingQueues = new TreeMap<String, ConcurrentLinkedQueue<IMessage>>(); ConcurrentLinkedQueue<IMessage> tempQueue = null; IMessage tempMessage = null; // Traverse the outgoing queue and put the messages with the same // dstVertexID into the same queue in the tree map. Iterator<IMessage> iter = outgoingQueue.iterator(); String dstVertexID = null; /**The result queue for return.*/ ConcurrentLinkedQueue<IMessage> resultQueue = new ConcurrentLinkedQueue<IMessage>(); while (iter.hasNext()) { tempMessage = iter.next(); dstVertexID = tempMessage.getDstVertexID(); tempQueue = outgoingQueues.get(dstVertexID); if (tempQueue == null) { tempQueue = new ConcurrentLinkedQueue<IMessage>(); } tempQueue.add(tempMessage); outgoingQueues.put(dstVertexID, tempQueue); } // Do combine operation for each of the outgoing queues. for (Entry<String, ConcurrentLinkedQueue<IMessage>> entry : outgoingQueues.entrySet()) { tempQueue = entry.getValue(); tempMessage = (IMessage) this.combiner.combine(tempQueue.iterator()); resultQueue.add(tempMessage); } outgoingQueue.clear(); outgoingQueues.clear(); return resultQueue; }
From source file:com.chinamobile.bcbsp.comm.CombinerTool.java
/** run method. */ public void run() { LOG.info("[CombinerTool] Start!"); long time = 0; long totalBeforeSize = 0; long totalAfterSize = 0; String outgoIndex = null;/*from w w w . j a va2 s . c om*/ ConcurrentLinkedQueue<IMessage> maxQueue = null; int maxSize = 0; int lastCount = 0; while (!this.sender.getNoMoreMessagesFlag()) { outgoIndex = this.messageQueues.getMaxOutgoingQueueIndex(); if (outgoIndex != null) { maxSize = this.messageQueues.getOutgoingQueueSize(outgoIndex); } lastCount = (this.combinedCountsMap.get(outgoIndex) == null ? 0 : this.combinedCountsMap.get(outgoIndex)); // If new updated size is over the threshold if ((maxSize - lastCount) > this.combineThreshold) { // Get the queue out of the map. maxQueue = this.messageQueues.removeOutgoingQueue(outgoIndex); if (maxQueue == null) { continue; } long start = System.currentTimeMillis(); /* Clock */ // Combine the messages. maxQueue = combine(maxQueue); long end = System.currentTimeMillis(); /* Clock */ time = time + end - start; int maxSizeBefore = maxSize; totalBeforeSize += maxSizeBefore; maxSize = maxQueue.size(); totalAfterSize += maxSize; // Note the count after combination. this.combinedCountsMap.put(outgoIndex, maxQueue.size()); // Put the combined messages back to the outgoing queue. Iterator<IMessage> iter = maxQueue.iterator(); while (iter.hasNext()) { this.messageQueues.outgoAMessage(outgoIndex, iter.next()); } maxQueue.clear(); } else { try { // Wait for 500ms until next check. Thread.sleep(500); } catch (Exception e) { throw new RuntimeException("[[CombinerTool] run exception", e); } } } LOG.info("[CombinerTool] has combined totally (" + totalBeforeSize + ") messages into (" + totalAfterSize + "). Compression rate = " + (float) totalAfterSize * 100 / totalBeforeSize + "%."); LOG.info("[CombinerTool] has used time: " + time / TIME_CHANGE + " seconds totally!"); LOG.info("[CombinerTool] Die!"); }
From source file:com.chinamobile.bcbsp.bspstaff.BSPStaff.java
@Override public void peerProcessing(BSPPeer peer, BSP bsp, BSPJob job, int superStepCounter, BSPStaffContext context, boolean activeFlag) { // /**Feng added for migrate staff messages*/ // ConcurrentLinkedQueue<IMessage> migrateMessages; String migrateMessages = null; StringBuffer sb = new StringBuffer(); /** Clock */// w w w .jav a 2 s .c o m long tmpStart = System.currentTimeMillis(); if (peer == null) { throw new RuntimeException("No key-value to compute for staff " + this.getSid()); } loadGraphTime = loadGraphTime + (System.currentTimeMillis() - tmpStart); // Get the incomed message queue for this peer(partition). try { ConcurrentLinkedQueue<IMessage> messages = this.communicator .getMessageQueue(String.valueOf(Constants.DEFAULT_PEER_DST_MESSSAGE_ID)); // Aggregate the new values for each vertex. Clock the time // cost. publishAggregateValues(context); LOG.info("ljn test : peerProcessing messages size is " + messages.size()); tmpStart = System.currentTimeMillis(); aggregate(messages, job, peer, this.currentSuperStepCounter); aggregateTime = aggregateTime + (System.currentTimeMillis() - tmpStart); context.updatePeer(peer); if (superStepCounter > 0) { if (!activeFlag && (messages.size() == 0)) { return; } } Iterator<IMessage> messagesIter = messages.iterator(); // Call the compute function for local computation. /* * Publish the total result aggregate values into the bsp's cache for the * user's function's accession in the next super step. */ // publishAggregateValues(context); /** Clock */ tmpStart = System.currentTimeMillis(); try { // LOG.info("bsp.compute vertex message!"); bsp.compute(messagesIter, context); } catch (Exception e) { throw new RuntimeException("catch exception", e); } computeTime = computeTime + (System.currentTimeMillis() - tmpStart); /** Clock */ messages.clear(); peer.resetPair(); /** Clock */ tmpStart = System.currentTimeMillis(); collectMsgsTime = collectMsgsTime + (System.currentTimeMillis() - tmpStart); } catch (Exception e) { throw new RuntimeException("catch exception on peer compute", e); } /** Clock */ }
From source file:com.chinamobile.bcbsp.bspstaff.BSPStaff.java
@Override public void vertexProcessing(Vertex v, BSP bsp, BSPJob job, int superStepCounter, BSPStaffContext context, boolean activeFlag) throws IOException { // /**Feng added for migrate staff messages*/ // ConcurrentLinkedQueue<IMessage> migrateMessages; String migrateMessages = null; StringBuffer sb = new StringBuffer(); /** Clock *///from www. j a v a 2 s . c o m // long tmpStart = System.currentTimeMillis(); if (v == null) { LOG.error("Fail to get the HeadNode of index[" + "] " + "and the system will skip the record"); return; } // loadGraphTime = loadGraphTime + (System.currentTimeMillis() - tmpStart); // Get the incomed message queue for this vertex. ConcurrentLinkedQueue<IMessage> messages = this.communicator .getMessageQueue(String.valueOf(v.getVertexID())); if (this.migratedStaffFlag == true) { // LOG.info("Feng test use the migrated messages! "+v.getVertexID()); messages = this.communicator// Feng test use the migrated messages .getMigrateMessageQueue(String.valueOf(v.getVertexID())); } // LOG.info("Feng test Messagesize! "+messages.size()); // if (messages.size() > 0) // LOG.info("<TEST1> V: " +v.getVertexID() +" MSG ID " + // messages.poll().getMessageId()); // Aggregate the new values for each vertex. Clock the time // cost. // tmpStart = System.currentTimeMillis(); aggregate(messages, job, v, this.currentSuperStepCounter); // aggregateTime = aggregateTime + (System.currentTimeMillis() - tmpStart); // Note Ever Fault. // Note Ever Edit /\2014-01-23 context.refreshVertex(v); if (superStepCounter > 0) { if (!activeFlag && (messages.size() == 0)) { return; } // for activemap update else { upadateActiveBitMap(v); } } if (this.openMigrateMode == true && messages.size() > 0) { // this.migrateMessages.clear(); // migrateMessages.addAll(messages); // LOG.info("Feng test messageSize "+messages.size()); // for(IMessage mig:messages){ // LOG.info("Feng test message "+mig.intoString()); // } Iterator<IMessage> iterator = messages.iterator(); sb.append(v.getVertexID().toString() + Constants.MESSAGE_SPLIT); while (iterator.hasNext()) { IMessage msg = iterator.next(); String msgID = msg.getMessageId().toString(); String msgValue = msg.getContent().toString(); if (msgID != null) { sb.append(msgID + Constants.SPLIT_FLAG + msgValue + Constants.SPACE_SPLIT_FLAG); } migrateMessages = sb.toString(); this.migrateMessagesString.add(migrateMessages); } // LOG.info("Feng test record migrate messages!"); } Iterator<IMessage> messagesIter = messages.iterator(); // Iterator<IMessage> messagesIterTest = messages.iterator(); // if(messagesIterTest.hasNext()){ // IMessage i = messagesIterTest.next(); // LOG.info("Feng test staff messages" +i.getMessageId() // +"messagesSize! "+i.getContent()); // } // LOG.info("Feng test! "+ ((PageRankMessage) // (messagesIterTest.next())).getContent()); // Call the compute function for local computation. /* * Publish the total result aggregate values into the bsp's cache for the * user's function's accession in the next super step. */ publishAggregateValues(context); /** Clock */ // tmpStart = System.currentTimeMillis(); try { // LOG.info("bsp.compute vertex message!"); bsp.compute(messagesIter, context); } catch (Exception e) { throw new RuntimeException("catch exception", e); } // computeTime = computeTime + (System.currentTimeMillis() - tmpStart); /** Clock */ messages.clear(); /** Clock */ // tmpStart = System.currentTimeMillis(); // collectMsgsTime = collectMsgsTime + (System.currentTimeMillis() - // tmpStart); /** Clock */ }