List of usage examples for java.util.concurrent LinkedBlockingQueue drainTo
public int drainTo(Collection<? super E> c)
From source file:org.apache.bigtop.bigpetstore.qstream.HttpLoadGen.java
/** * Appends via REST calls./*from w w w . j a v a 2 s . co m*/ */ public LinkedBlockingQueue<Transaction> startWriteQueue(final int milliseconds) { /** * Write queue. Every 5 seconds, write */ final LinkedBlockingQueue<Transaction> transactionQueue = new LinkedBlockingQueue<Transaction>( getQueueSize()); new Thread() { @Override public void run() { int fileNumber = 0; while (true) { waitFor(milliseconds, transactionQueue); System.out.println("CLEARING " + transactionQueue.size() + " elements from queue."); Stack<Transaction> transactionsToWrite = new Stack<Transaction>(); transactionQueue.drainTo(transactionsToWrite); /** * pop transactions from the queue, and sent them over http as json. */ while (!transactionsToWrite.isEmpty()) { try { String trAsJson = URLEncoder.encode(Utils.toJson(transactionsToWrite.pop())); /** * i.e. wget http://localhost:3000/rpush/guestbook/{"name":"cos boudnick", "state":"...",...} */ HttpResponse resp = Utils.get(path + "/" + trAsJson); if (total % 20 == 0) System.out.println("wrote customer " + trAsJson); total++; } catch (Throwable t) { System.err.println("transaction failed.... !"); t.printStackTrace(); } System.out.println("TRANSACTIONS SO FAR " + total++ + " RATE " + total / ((System.currentTimeMillis() - startTime) / 1000)); } } } }.start(); return transactionQueue; }
From source file:org.commoncrawl.service.pagerank.slave.PageRankUtils.java
public static void distributeRank(final PRValueMap valueMap, final Path outlinksFile, final boolean outlinksIsRemote, File localOutputDir, String remoteOutputDir, int thisNodeIdx, int nodeCount, int iterationNumber, final ProgressAndCancelCheckCallback progressCallback) throws IOException { final Configuration conf = CrawlEnvironment.getHadoopConfig(); Vector<PRValueOutputStream> outputStreamVector = new Vector<PRValueOutputStream>(); // allocate a queue ... final LinkedBlockingQueue<OutlinkItem> queue = new LinkedBlockingQueue<OutlinkItem>(20000); try {/*from w w w .j av a 2 s .c o m*/ // start the loader thread ... Thread loaderThread = new Thread(new Runnable() { final BytesWritable key = new BytesWritable(); final BytesWritable value = new BytesWritable(); final DataInputBuffer keyStream = new DataInputBuffer(); final DataInputBuffer valueStream = new DataInputBuffer(); @Override public void run() { LOG.info("Opening Outlinks File at:" + outlinksFile); SequenceFile.Reader reader = null; try { FileSystem fsForOutlinksFile = null; if (outlinksIsRemote) { fsForOutlinksFile = CrawlEnvironment.getDefaultFileSystem(); } else { fsForOutlinksFile = FileSystem.getLocal(conf); } FileStatus outlinksFileStatus = fsForOutlinksFile.getFileStatus(outlinksFile); long bytesToReadTotal = (outlinksFileStatus != null) ? outlinksFileStatus.getLen() : 0; reader = new SequenceFile.Reader(fsForOutlinksFile, outlinksFile, conf); OutlinkItem item = new OutlinkItem(); int itemCount = 0; boolean isCancelled = false; while (!isCancelled && reader.next(key, value)) { keyStream.reset(key.getBytes(), 0, key.getLength()); valueStream.reset(value.getBytes(), 0, value.getLength()); //populate item from data readURLFPFromStream(keyStream, item.targetFingerprint); item.urlCount = readURLFPAndCountFromStream(valueStream, item.sourceFingerprint); try { long blockTimeStart = System.currentTimeMillis(); queue.put(item); long blockTimeEnd = System.currentTimeMillis(); } catch (InterruptedException e) { } item = new OutlinkItem(); if (itemCount++ % 10000 == 0 && progressCallback != null) { float percentComplete = (float) reader.getPosition() / (float) bytesToReadTotal; if (progressCallback.updateProgress(percentComplete)) { LOG.info("Cancel check callback returned true.Cancelling outlink item load"); isCancelled = true; } } } item.sourceFingerprint = null; item.targetFingerprint = null; // add empty item try { if (!isCancelled) { queue.put(item); } else { queue.put(new OutlinkItem(new IOException("Operation Cancelled"))); } } catch (InterruptedException e) { } } catch (IOException e) { // add error item to queue. try { queue.put(new OutlinkItem(e)); } catch (InterruptedException e1) { } } finally { if (reader != null) try { reader.close(); } catch (IOException e) { } } } }); loaderThread.start(); // first things first ... initialize output stream vector FileSystem fileSystem = buildDistributionOutputStreamVector(true, getOutlinksBaseName(thisNodeIdx, iterationNumber), localOutputDir, remoteOutputDir, thisNodeIdx, nodeCount, outputStreamVector); try { // open outlinks file . LOG.info("Iterating Items in Outlinks File and Writing Test Value"); int itemCount = 0; int totalOutlinkCount = 0; int iterationOutlinkCount = 0; long iterationStart = System.currentTimeMillis(); long timeStart = iterationStart; boolean done = false; ArrayList<OutlinkItem> items = new ArrayList<OutlinkItem>(); // start iterating outlinks while (!done) { //OutlinkItem item = null; //try { long waitTimeStart = System.currentTimeMillis(); queue.drainTo(items); long waitTimeEnd = System.currentTimeMillis(); //} catch (InterruptedException e) { //} for (OutlinkItem item : items) { if (item.error != null) { LOG.info( "Loader Thread Returned Error:" + CCStringUtils.stringifyException(item.error)); throw item.error; } else if (item.sourceFingerprint == null) { LOG.info("Loader Thread Indicated EOF via emtpy item"); done = true; } else { ++itemCount; /* LOG.info("SourceFP-DomainHash:" + item.sourceFingerprint.getDomainHash() + " URLHash:" + item.sourceFingerprint.getUrlHash() + " PartitionIdx:" + ((item.sourceFingerprint.hashCode() & Integer.MAX_VALUE) % CrawlEnvironment.PR_NUMSLAVES) ); */ // now get pr value for fingerprint (random seek in memory here!!!) float prValue = valueMap.getPRValue(item.sourceFingerprint) / (float) Math.max(item.urlCount, 1); // write value out int nodeIndex = (item.targetFingerprint.hashCode() & Integer.MAX_VALUE) % nodeCount; outputStreamVector.get(nodeIndex).writePRValue(item.targetFingerprint, item.sourceFingerprint, prValue); if (itemCount % 10000 == 0) { long timeEnd = System.currentTimeMillis(); int milliseconds = (int) (timeEnd - iterationStart); LOG.info("Distribute PR for 10000 Items with:" + iterationOutlinkCount + " Outlinks Took:" + milliseconds + " Milliseconds" + " QueueCount:" + queue.size()); iterationStart = System.currentTimeMillis(); totalOutlinkCount += iterationOutlinkCount; iterationOutlinkCount = 0; } } } items.clear(); } totalOutlinkCount += iterationOutlinkCount; LOG.info("Distribute Finished for a total of:" + itemCount + " Items with:" + totalOutlinkCount + " Outlinks Took:" + (System.currentTimeMillis() - timeStart) + " Milliseconds"); LOG.info("Waiting for Loader Thread to Die"); try { loaderThread.join(); } catch (InterruptedException e) { } LOG.info("Loader Thread Died - Moving on..."); } finally { for (PRValueOutputStream info : outputStreamVector) { if (info != null) { info.close(false); } } if (fileSystem != null) { fileSystem.close(); } } } catch (IOException e) { LOG.error("Exception caught while distributing outlinks:" + CCStringUtils.stringifyException(e)); throw e; } }
From source file:org.rifidi.edge.adapter.alien.Alien9800Reader.java
public synchronized boolean applyPropertyChanges( LinkedBlockingQueue<AlienCommandObjectWrapper> propCommandsToBeExecuted, boolean block) { // TODO: may need to synchnonize the hashmap before I clear it? Alien9800ReaderSession aliensession = session.get(); if (aliensession != null) { ArrayList<AlienCommandObjectWrapper> commands = new ArrayList<AlienCommandObjectWrapper>(); propCommandsToBeExecuted.drainTo(commands); AlienPropertyCommand command = new AlienPropertyCommand("", readerProperties, commands); if (block) { return aliensession.submitAndBlock(command, 10, TimeUnit.SECONDS); } else {/* w w w .j a va 2 s .c o m*/ aliensession.submit(command); return true; } } return false; }