List of usage examples for java.util.concurrent ConcurrentLinkedQueue size
public int size()
From source file:com.chinamobile.bcbsp.comm.RPCSender.java
/** Run method of Sender Thread. */ public void run() { try {/*from www.j av a 2 s. co m*/ LOG.info("[RPCSender] starts successfully!"); this.initialize(); synchronized (this) { try { while (!this.condition) { wait(); } this.condition = false; } catch (InterruptedException e) { LOG.error("[RPCSender] caught: ", e); } } do { LOG.info("[RPCSender] begins to work for super step <" + this.superStepCounter + ">!"); LOG.info("this.messageQueues.getNextOutgoingQueueIndex(); " + this.messageQueues.getNextOutgoingQueueIndex()); this.connectTime = 0L; /** Clock */ this.sendTime = 0L; /** Clock */ RPCSingleSendSlave slave = null; if (combinerFlag && !this.noMoreMessagesFlag) { this.combinerTool = new CombinerTool(this, messageQueues, combiner, combineThreshold); this.combinerTool.start(); } String outgoingIndex = null; ConcurrentLinkedQueue<IMessage> maxQueue = null; int maxSize = 0; while (true) { // and no more messages will come. maxSize = 0; maxQueue = null; // outgoingIndex = this.messageQueues.getMaxOutgoingQueueIndex(); outgoingIndex = this.messageQueues.getNextOutgoingQueueIndex(); if (outgoingIndex != null) { maxSize = this.messageQueues.getOutgoingQueueSize(outgoingIndex); } // When the whole outgoingQueues are empty and no more messages will // come, exit while. if (outgoingIndex == null) { if (this.noMoreMessagesFlag) { if (this.messageQueues.getOutgoingQueuesSize() > 0) { continue; } if (!this.combinerFlag) { /** no more messages and no combiner set */ LOG.info("[RPCSender] exits while for no more messages " + "and no combiner set."); break; } else { if (this.combinerTool == null) { /** no more messages and no combiner created */ LOG.info("[RPCSender] exits while for no more messages " + "and no combiner created."); break; } else if (!this.combinerTool.isAlive()) { /** no more messages and combiner has exited */ LOG.info("[Sender] exits while for no more messages " + "and combiner has exited."); break; } } } else { try { Thread.sleep(500); } catch (Exception e) { LOG.error("[Sender] caught: ", e); } continue; } } else if (maxSize > this.sendThreshold || this.noMoreMessagesFlag) { // Get a producer tool to send the maxQueue slave = this.rpcSlavePool.getSlave(outgoingIndex, this.superStepCounter, this); if (slave.isIdle()) { // Get the queue out of the map. maxQueue = this.messageQueues.removeOutgoingQueue(outgoingIndex); if (maxQueue == null) { continue; } maxSize = maxQueue.size(); // add by chen for (IMessage m : maxQueue) { this.messageBytesCount += m.size(); } slave.setPackSize(this.packSize); LOG.info("current slave id is " + slave.getId()); LOG.info("NOW senderSlave will sender messageQueue and size = " + maxSize); slave.setStaffId(this.staffId); slave.addMessages(maxQueue); this.messageCount += maxSize; // Set the producer's state to busy to start it. slave.setIdle(false); } if (slave.isFailed()) { this.messageQueues.removeOutgoingQueue(outgoingIndex); } } } // while LOG.info("[RPCSender] has started " + this.rpcSlavePool.getSlaveCount() + " slaves totally."); // Tell all producers that no more messages. this.rpcSlavePool.finishAll(); // sender??????? while (true) { // Wait for the producers finish message sending. int running = this.rpcSlavePool.getActiveSlaveCount(this.superStepCounter); LOG.info("[RPCSender] There are still <" + running + "> RPCSendSlaves alive."); if (running == 0) { break; } try { Thread.sleep(500); } catch (Exception e) { LOG.error("[Sender] caught: ", e); } } // while // add by chen this.communicator.setCombineOutgoMessageCounter(this.messageCount); this.communicator.setCombineOutgoMessageBytesCounter(messageBytesCount); LOG.info("[RPCSender] has sent " + this.messageCount + " messages totally for super step <" + this.superStepCounter + ">! And enter waiting......"); LOG.info("[==>Clock<==] <RPCSender's create connection> used " + this.connectTime / TIME_CHANGE + " seconds"); /** Clock */ LOG.info("[==>Clock<==] <RPCSender's send messages> used " + this.sendTime / TIME_CHANGE + " seconds"); /** Clock */ this.over = true; synchronized (this) { try { while (!this.condition) { wait(); } this.condition = false; } catch (InterruptedException e) { LOG.error("[RPCSender] caught: ", e); } } } while (!this.completed); LOG.info("[PRCSender] exits."); } catch (Exception e) { LOG.error("RPCSender caught: ", e); } }
From source file:dendroscope.autumn.hybridnumber.ComputeHybridNumber.java
/** * recursively compute the hybrid number * * @param root1/*from w w w .j a v a 2s . c o m*/ * @param root2 * @param isReduced @return hybrid number * @param retry * @param topLevel * @param scoreAbove * @param additionalAbove */ private int computeHybridNumberRec(final Root root1, final Root root2, boolean isReduced, Integer previousHybrid, BitSet retry, final boolean topLevel, final int scoreAbove, final ValuesList additionalAbove) throws IOException, CanceledException { if (System.currentTimeMillis() > nextTime) { synchronized (progressListener) { nextTime += waitTime; waitTime *= 1.5; progressListener.incrementProgress(); } } else progressListener.checkForCancel(); // System.err.println("computeHybridNumberRec: tree1=" + Basic.toString(root1.getTaxa()) + " tree2=" + Basic.toString(root2.getTaxa())); // root1.reorderSubTree(); // root2.reorderSubTree(); if (checking) { root1.checkTree(); root2.checkTree(); } BitSet taxa = root1.getTaxa(); String key = root1.toStringTreeSparse() + root2.toStringTreeSparse(); // System.err.println("Key: "+key); Integer value; synchronized (lookupTable) { value = (Integer) lookupTable.get(key); if (value != null) return value; } if (!root2.getTaxa().equals(taxa)) throw new RuntimeException("Unequal taxon sets: X=" + Basic.toString(root1.getTaxa()) + " vs " + Basic.toString(root2.getTaxa())); if (!isReduced) { switch (SubtreeReduction.apply(root1, root2, null)) { case ISOMORPHIC: synchronized (lookupTable) { lookupTable.put(key, 0); } if (topLevel) { bestScore.lowerTo(0); progressListener.setSubtask("Best score: " + bestScore); } return 0; // two trees are isomorphic, no hybrid node needed case REDUCED: // a reduction was performed, cannot maintain lexicographical ordering in removal loop below previousHybrid = null; break; case IRREDUCIBLE: break; } Single<Integer> placeHolderTaxa = new Single<Integer>(); final Pair<Root, Root> clusterTrees = ClusterReduction.apply(root1, root2, placeHolderTaxa); final boolean retryTop = false && (previousHybrid != null && placeHolderTaxa.get() < previousHybrid); // if the taxa involved in the cluster reduction come before the previously removed hybrid, do full retry // retryTop doesn't work final BitSet fRetry = retry; if (clusterTrees != null) // will perform cluster-reduction { final Value score1 = new Value(0); final Value score2 = new Value(1); // because the cluster could not be reduced using an subtree reduction, can assume that we will need one reticulation for this final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false); if (verbose) System.err.println("Starting parallel loop"); final CountDownLatch countDownLatch = new CountDownLatch(2); final Integer fPrevious = previousHybrid; // setup task: final Task task1 = new Task(); // first of two cluster-reduction tasks task1.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on cluster-reduction"); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } final ValuesList additionalAbove1 = additionalAbove.copyWithAdditionalElement(score2); if (scoreAbove + additionalAbove1.sum() < bestScore.get()) { int h = computeHybridNumberRec(root1, root2, false, fPrevious, fRetry, false, scoreAbove, additionalAbove1); score1.set(h); } else { score1.set(LARGE); } additionalAbove1.clear(); } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); final Task task2 = new Task(); // second of two cluster-reduction tasks task2.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on cluster-reduction"); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } final ValuesList additionalAbove2 = additionalAbove.copyWithAdditionalElement(score1); if (scoreAbove + additionalAbove2.sum() < bestScore.get()) { int h = computeHybridNumberRec(clusterTrees.getFirst(), clusterTrees.getSecond(), true, fPrevious, fRetry, false, scoreAbove, additionalAbove2); score2.set(h); } else { score2.set(LARGE); } additionalAbove2.clear(); } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); // start a task in this thread scheduledThreadPoolExecutor.execute(task1); task2.run(); task1.run(); // try to run task1 in current thread if it hasn't yet started execution. If the task is already running or has completed, will simply return try { if (verbose) System.err.println("waiting..."); // wait until all tasks have completed countDownLatch.await(); if (verbose) System.err.println("done"); } catch (InterruptedException e) { Basic.caught(e); } clusterTrees.getFirst().deleteSubTree(); clusterTrees.getSecond().deleteSubTree(); int total = scoreAbove + additionalAbove.sum() + score1.get() + score2.get(); if (topLevel && (total < bestScore.get())) // score above will be zero, but put this here anyway to avoid confusion { bestScore.lowerTo(total); progressListener.setSubtask("Current best score: " + bestScore); } synchronized (lookupTable) { Integer old = (Integer) lookupTable.get(key); if (old == null || total < old) lookupTable.put(key, total); } return score1.get() + score2.get(); } } List<Root> leaves1 = root1.getAllLeaves(); if (leaves1.size() <= 2) // try 2 rather than one... { return 0; } final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false); if (verbose) System.err.println("Starting parallel loop"); final CountDownLatch countDownLatch = new CountDownLatch(leaves1.size()); final Value bestSubH = new Value(LARGE); // schedule all tasks to be performed final ConcurrentLinkedQueue<Task> queue = new ConcurrentLinkedQueue<Task>(); for (Node leaf2remove : leaves1) { final BitSet taxa2remove = ((Root) leaf2remove).getTaxa(); if (previousHybrid == null || previousHybrid < taxa2remove.nextSetBit(0)) { if (scoreAbove + additionalAbove.sum() + 1 >= bestScore.get()) return LARGE; // other thread has found a better result, abort // setup task: final Task task = new Task(); task.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on " + Basic.toString(taxa2remove)); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } queue.remove(task); if (scoreAbove + additionalAbove.sum() + 1 < bestScore.get()) { Root tree1X = CopyWithTaxaRemoved.apply(root1, taxa2remove); Root tree2X = CopyWithTaxaRemoved.apply(root2, taxa2remove); Refine.apply(tree1X, tree2X); int scoreBelow = computeHybridNumberRec(tree1X, tree2X, false, taxa2remove.nextSetBit(0), null, false, scoreAbove + 1, additionalAbove) + 1; if (topLevel && scoreBelow < bestScore.get()) { bestScore.lowerTo(scoreBelow); progressListener.setSubtask("Current best score: " + bestScore); } synchronized (bestSubH) { if (scoreBelow < bestSubH.get()) bestSubH.set(scoreBelow); } tree1X.deleteSubTree(); tree2X.deleteSubTree(); } } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); queue.add(task); } else // no task for this item, count down { countDownLatch.countDown(); progressListener.checkForCancel(); } } // grab one task for the current thread: Task taskForCurrentThread = queue.size() > 0 ? queue.poll() : null; // launch all others in the executor for (Task task : queue) scheduledThreadPoolExecutor.execute(task); // start a task in this thread if (taskForCurrentThread != null) taskForCurrentThread.run(); // try to run other tasks from the queue. Note that any task that is already running will return immediately while (queue.size() > 0) { Task task = queue.poll(); if (task != null) task.run(); } try { if (verbose) System.err.println("waiting..."); // wait until all tasks have completed countDownLatch.await(); if (verbose) System.err.println("done"); } catch (InterruptedException e) { Basic.caught(e); return LARGE; } // return the best value synchronized (lookupTable) { Integer old = (Integer) lookupTable.get(key); if (old == null || old > bestSubH.get()) lookupTable.put(key, bestSubH.get()); } return bestSubH.get(); }
From source file:com.chinamobile.bcbsp.comm.Sender.java
/** Run method of Sender Thread. */ public void run() { try {/* w w w .jav a 2 s . c om*/ LOG.info("[Sender] starts successfully!"); this.initialize(); synchronized (this) { try { while (!this.condition) { wait(); // wait for first begin. } this.condition = false; } catch (InterruptedException e) { LOG.error("[Sender] caught: ", e); } } do { // while for totally complete for this staff. LOG.info("[Sender] begins to work for sueper step <" + this.superStepCounter + ">!"); this.connectTime = 0L; /* Clock */ this.sendTime = 0L; ProducerTool producer = null; if (combinerFlag && !this.noMoreMessagesFlag) { this.combinerTool = new CombinerTool(this, messageQueues, combiner, combineThreshold); this.combinerTool.start(); } String outgoingIndex = null; ConcurrentLinkedQueue<IMessage> maxQueue = null; int maxSize = 0; while (true) { // Keep sending until the whole outgoingQueues is empty maxSize = 0; maxQueue = null; // outgoingIndex = this.messageQueues.getMaxOutgoingQueueIndex(); outgoingIndex = this.messageQueues.getNextOutgoingQueueIndex(); if (outgoingIndex != null) { maxSize = this.messageQueues.getOutgoingQueueSize(outgoingIndex); } // When the whole outgoingQueues are empty and no more messages will // come, exit while. if (outgoingIndex == null) { if (this.noMoreMessagesFlag) { if (this.messageQueues.getOutgoingQueuesSize() > 0) { continue; } if (!this.combinerFlag) { /** no more messages and no combiner set */ LOG.info("[Sender] exits while for no more messages " + "and no combiner set."); break; } else { if (this.combinerTool == null) { /* no more messages and no combiner created */ LOG.info("[Sender] exits while for no more messages " + "and no combiner created."); break; } else if (!this.combinerTool.isAlive()) { /* no more messages and combiner has exited */ LOG.info("[Sender] exits while for no more messages " + "and combiner has exited."); break; } } } else { try { Thread.sleep(500); } catch (Exception e) { LOG.error("[Sender] caught: ", e); } continue; } } else if (maxSize > this.sendThreshold || this.noMoreMessagesFlag) { // ????????? // Get a producer tool to send the maxQueue producer = this.producerPool.getProducer(outgoingIndex, this.superStepCounter, this); if (producer.isIdle()) { // Get the queue out of the map. maxQueue = this.messageQueues.removeOutgoingQueue(outgoingIndex); if (maxQueue == null) { continue; } maxSize = maxQueue.size(); // add by chen for (IMessage m : maxQueue) { this.messageBytesCount += m.size(); } producer.setPackSize(this.packSize); producer.addMessages(maxQueue); this.messageCount += maxSize; // Set the producer's state to busy to start it. producer.setIdle(false); } if (producer.isFailed()) { this.messageQueues.removeOutgoingQueue(outgoingIndex); } } } // while LOG.info("[Sender] has started " + producerPool.getProducerCount() + " producers totally."); // Tell all producers that no more messages. this.producerPool.finishAll(); while (true) { // Wait for the producers finish message sending. int running = this.producerPool.getActiveProducerCount(this.superStepCounter); LOG.info("[Sender] There are still <" + running + "> ProducerTools alive."); if (running == 0) { break; } try { Thread.sleep(500); } catch (Exception e) { LOG.error("[Sender] caught: ", e); } } // while // add by chen this.communicator.setCombineOutgoMessageCounter(this.messageCount); this.communicator.setCombineOutgoMessageBytesCounter(messageBytesCount); LOG.info("[Sender] has sent " + this.messageCount + " messages totally for super step <" + this.superStepCounter + ">! And enter waiting......"); LOG.info("[==>Clock<==] <Sender's create connection> used " + this.connectTime / TIME_CHANGE + " seconds"); /* Clock */ LOG.info("[==>Clock<==] <Sender's send messages> used " + this.sendTime / TIME_CHANGE + " seconds"); this.over = true; synchronized (this) { try { while (!this.condition) { wait(); } this.condition = false; } catch (InterruptedException e) { LOG.error("[Sender] caught: ", e); } } } while (!this.completed); LOG.info("[Sender] exits."); } catch (Exception e) { LOG.error("Sender caught: ", e); } }
From source file:com.chinamobile.bcbsp.bspstaff.BSPStaff.java
@Override public void peerProcessing(BSPPeer peer, BSP bsp, BSPJob job, int superStepCounter, BSPStaffContext context, boolean activeFlag) { // /**Feng added for migrate staff messages*/ // ConcurrentLinkedQueue<IMessage> migrateMessages; String migrateMessages = null; StringBuffer sb = new StringBuffer(); /** Clock */// w ww. jav a2 s. c o m long tmpStart = System.currentTimeMillis(); if (peer == null) { throw new RuntimeException("No key-value to compute for staff " + this.getSid()); } loadGraphTime = loadGraphTime + (System.currentTimeMillis() - tmpStart); // Get the incomed message queue for this peer(partition). try { ConcurrentLinkedQueue<IMessage> messages = this.communicator .getMessageQueue(String.valueOf(Constants.DEFAULT_PEER_DST_MESSSAGE_ID)); // Aggregate the new values for each vertex. Clock the time // cost. publishAggregateValues(context); LOG.info("ljn test : peerProcessing messages size is " + messages.size()); tmpStart = System.currentTimeMillis(); aggregate(messages, job, peer, this.currentSuperStepCounter); aggregateTime = aggregateTime + (System.currentTimeMillis() - tmpStart); context.updatePeer(peer); if (superStepCounter > 0) { if (!activeFlag && (messages.size() == 0)) { return; } } Iterator<IMessage> messagesIter = messages.iterator(); // Call the compute function for local computation. /* * Publish the total result aggregate values into the bsp's cache for the * user's function's accession in the next super step. */ // publishAggregateValues(context); /** Clock */ tmpStart = System.currentTimeMillis(); try { // LOG.info("bsp.compute vertex message!"); bsp.compute(messagesIter, context); } catch (Exception e) { throw new RuntimeException("catch exception", e); } computeTime = computeTime + (System.currentTimeMillis() - tmpStart); /** Clock */ messages.clear(); peer.resetPair(); /** Clock */ tmpStart = System.currentTimeMillis(); collectMsgsTime = collectMsgsTime + (System.currentTimeMillis() - tmpStart); } catch (Exception e) { throw new RuntimeException("catch exception on peer compute", e); } /** Clock */ }
From source file:com.chinamobile.bcbsp.bspstaff.BSPStaff.java
@Override public void vertexProcessing(Vertex v, BSP bsp, BSPJob job, int superStepCounter, BSPStaffContext context, boolean activeFlag) throws IOException { // /**Feng added for migrate staff messages*/ // ConcurrentLinkedQueue<IMessage> migrateMessages; String migrateMessages = null; StringBuffer sb = new StringBuffer(); /** Clock *///from w ww.j a va2 s . co m // long tmpStart = System.currentTimeMillis(); if (v == null) { LOG.error("Fail to get the HeadNode of index[" + "] " + "and the system will skip the record"); return; } // loadGraphTime = loadGraphTime + (System.currentTimeMillis() - tmpStart); // Get the incomed message queue for this vertex. ConcurrentLinkedQueue<IMessage> messages = this.communicator .getMessageQueue(String.valueOf(v.getVertexID())); if (this.migratedStaffFlag == true) { // LOG.info("Feng test use the migrated messages! "+v.getVertexID()); messages = this.communicator// Feng test use the migrated messages .getMigrateMessageQueue(String.valueOf(v.getVertexID())); } // LOG.info("Feng test Messagesize! "+messages.size()); // if (messages.size() > 0) // LOG.info("<TEST1> V: " +v.getVertexID() +" MSG ID " + // messages.poll().getMessageId()); // Aggregate the new values for each vertex. Clock the time // cost. // tmpStart = System.currentTimeMillis(); aggregate(messages, job, v, this.currentSuperStepCounter); // aggregateTime = aggregateTime + (System.currentTimeMillis() - tmpStart); // Note Ever Fault. // Note Ever Edit /\2014-01-23 context.refreshVertex(v); if (superStepCounter > 0) { if (!activeFlag && (messages.size() == 0)) { return; } // for activemap update else { upadateActiveBitMap(v); } } if (this.openMigrateMode == true && messages.size() > 0) { // this.migrateMessages.clear(); // migrateMessages.addAll(messages); // LOG.info("Feng test messageSize "+messages.size()); // for(IMessage mig:messages){ // LOG.info("Feng test message "+mig.intoString()); // } Iterator<IMessage> iterator = messages.iterator(); sb.append(v.getVertexID().toString() + Constants.MESSAGE_SPLIT); while (iterator.hasNext()) { IMessage msg = iterator.next(); String msgID = msg.getMessageId().toString(); String msgValue = msg.getContent().toString(); if (msgID != null) { sb.append(msgID + Constants.SPLIT_FLAG + msgValue + Constants.SPACE_SPLIT_FLAG); } migrateMessages = sb.toString(); this.migrateMessagesString.add(migrateMessages); } // LOG.info("Feng test record migrate messages!"); } Iterator<IMessage> messagesIter = messages.iterator(); // Iterator<IMessage> messagesIterTest = messages.iterator(); // if(messagesIterTest.hasNext()){ // IMessage i = messagesIterTest.next(); // LOG.info("Feng test staff messages" +i.getMessageId() // +"messagesSize! "+i.getContent()); // } // LOG.info("Feng test! "+ ((PageRankMessage) // (messagesIterTest.next())).getContent()); // Call the compute function for local computation. /* * Publish the total result aggregate values into the bsp's cache for the * user's function's accession in the next super step. */ publishAggregateValues(context); /** Clock */ // tmpStart = System.currentTimeMillis(); try { // LOG.info("bsp.compute vertex message!"); bsp.compute(messagesIter, context); } catch (Exception e) { throw new RuntimeException("catch exception", e); } // computeTime = computeTime + (System.currentTimeMillis() - tmpStart); /** Clock */ messages.clear(); /** Clock */ // tmpStart = System.currentTimeMillis(); // collectMsgsTime = collectMsgsTime + (System.currentTimeMillis() - // tmpStart); /** Clock */ }
From source file:spade.storage.Neo4j.java
public static void index(String dbpath, boolean printProgress) { int totalThreads = Runtime.getRuntime().availableProcessors(); final ConcurrentLinkedQueue<Node> nodeTaskQueue = new ConcurrentLinkedQueue<Node>(); final ConcurrentLinkedQueue<Relationship> edgeTaskQueue = new ConcurrentLinkedQueue<Relationship>(); final ReentrantReadWriteLock nodeRwlock = new ReentrantReadWriteLock(); final ReentrantReadWriteLock edgeRwlock = new ReentrantReadWriteLock(); final Index<Node> vertexIndex; final RelationshipIndex edgeIndex; System.out.println("Loading database..."); File databaseFile = new File(dbpath); final GraphDatabaseService graphDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(databaseFile) .setConfig(GraphDatabaseSettings.pagecache_memory, "" + (Runtime.getRuntime().totalMemory() * 9) / 10) // .setConfig(GraphDatabaseSettings.keep_logical_logs, "false") .newGraphDatabase();//from w w w.ja va 2 s. co m System.out.println("Loaded"); // clear already present indexes try (Transaction tx = graphDb.beginTx()) { graphDb.index().forNodes(spade.storage.Neo4j.VERTEX_INDEX).delete(); tx.success(); } try (Transaction tx = graphDb.beginTx()) { graphDb.index().forRelationships(spade.storage.Neo4j.EDGE_INDEX).delete(); tx.success(); } // System.out.println("Creating Indexing discriptors..."); try (Transaction tx = graphDb.beginTx()) { vertexIndex = graphDb.index().forNodes(spade.storage.Neo4j.VERTEX_INDEX); tx.success(); } try (Transaction tx = graphDb.beginTx()) { edgeIndex = graphDb.index().forRelationships(spade.storage.Neo4j.EDGE_INDEX); tx.success(); } System.out.println("Created"); class NodeIndexer implements Runnable { public void run() { Transaction tx = graphDb.beginTx(); int counter = 0; try { while (!Thread.currentThread().isInterrupted()) { if (counter < 10000) { Node node = nodeTaskQueue.poll(); if (node == null) { continue; } for (String key : node.getPropertyKeys()) { vertexIndex.add(node, key, (String) node.getProperty(key)); } node.setProperty(ID_STRING, node.getId()); vertexIndex.add(node, ID_STRING, Long.toString(node.getId())); counter++; } if (counter > 1000 && nodeRwlock.writeLock().tryLock()) { tx.success(); tx.close(); tx = graphDb.beginTx(); nodeRwlock.writeLock().unlock(); counter = 0; } } } finally { // tx.success(); tx.close(); if (nodeRwlock.writeLock().isHeldByCurrentThread()) { nodeRwlock.writeLock().unlock(); } } } } class RelationshipIndexer implements Runnable { public void run() { Transaction tx = graphDb.beginTx(); int counter = 0; try { while (!Thread.currentThread().isInterrupted()) { if (counter < 10000) { Relationship relationship = edgeTaskQueue.poll(); if (relationship == null) { continue; } for (String key : relationship.getPropertyKeys()) { edgeIndex.add(relationship, key, (String) relationship.getProperty(key)); } relationship.setProperty(ID_STRING, relationship.getId()); edgeIndex.add(relationship, ID_STRING, Long.toString(relationship.getId())); counter++; } if (counter > 1000 && edgeRwlock.writeLock().tryLock()) { // tx.success(); tx.close(); tx = graphDb.beginTx(); edgeRwlock.writeLock().unlock(); counter = 0; } } } finally { // tx.success(); tx.close(); if (edgeRwlock.writeLock().isHeldByCurrentThread()) { edgeRwlock.writeLock().unlock(); } } } } ArrayList<Thread> nodeWorkers = new ArrayList<Thread>(); for (int i = 0; i < totalThreads / 2; i++) { Thread th = new Thread(new NodeIndexer()); nodeWorkers.add(th); th.start(); } ArrayList<Thread> edgeWorkers = new ArrayList<Thread>(); for (int i = 0; i < totalThreads / 2; i++) { Thread th = new Thread(new RelationshipIndexer()); edgeWorkers.add(th); th.start(); } System.out.println("Counted Nodes and Relationships to index..."); final long total; try (Transaction tx = graphDb.beginTx()) { total = Iterators.count(graphDb.getAllNodes().iterator()) + Iterators.count(graphDb.getAllRelationships().iterator()); tx.success(); } System.out.println("done.\n"); long percentageCompleted = 0; int count = 0; try (Transaction tx = graphDb.beginTx()) { // index nodes Iterator<Node> nodeIterator = graphDb.getAllNodes().iterator(); Iterator<Relationship> edgeIterator = graphDb.getAllRelationships().iterator(); while (edgeIterator.hasNext() || nodeIterator.hasNext()) { if (nodeIterator.hasNext() && nodeTaskQueue.size() < 10000) { nodeTaskQueue.add(nodeIterator.next()); count = count + 1; } if (edgeIterator.hasNext() && edgeTaskQueue.size() < 10000) { edgeTaskQueue.add(edgeIterator.next()); count = count + 1; } if (printProgress) { if (((count * 100) / total) > percentageCompleted) { Runtime rt = Runtime.getRuntime(); long totalMemory = rt.totalMemory() / 1024 / 1024; long freeMemory = rt.freeMemory() / 1024 / 1024; long usedMemory = totalMemory - freeMemory; System.out.print("| Cores: " + rt.availableProcessors() + " | Threads: " + totalThreads + " | Heap (MB) - total: " + totalMemory + " , " + (freeMemory * 100) / totalMemory + "% free" // + " | Total Objects (nodes + relationships) to Index: " + total + " | Indexing Object (nodes + relationships): " + count + " / " + total + " | Completed: " + percentageCompleted + " %" + " |\r"); } percentageCompleted = (count * 100) / total; } } tx.success(); } System.out.println("\n\nIndexing completed. Waiting for queues to clear..."); try { while (nodeTaskQueue.size() != 0 || edgeTaskQueue.size() != 0) { Thread.sleep(1000); } } catch (InterruptedException exception) { } System.out.println("Queues cleared. Threads teardown started..."); for (int i = 0; i < totalThreads / 2; i++) { nodeWorkers.get(i).interrupt(); try { nodeWorkers.get(i).join(); } catch (InterruptedException exception) { } } for (int i = 0; i < totalThreads / 2; i++) { edgeWorkers.get(i).interrupt(); try { edgeWorkers.get(i).join(); } catch (InterruptedException exception) { } } System.out.println("Database shutdown started..."); graphDb.shutdown(); }