List of usage examples for java.util.concurrent Semaphore release
public void release(int permits)
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
private static void launchInTestMode() { File baseTestDir = new File("/tmp/logManagerTest"); FileUtils.recursivelyDeleteFile(baseTestDir); baseTestDir.mkdir();/*from www .ja v a 2 s . com*/ File remoteDir = new File(baseTestDir, "remote"); File localDir = new File(baseTestDir, "local"); remoteDir.mkdir(); localDir.mkdir(); final TreeMap<String, URLFP> urlToFPMap = new TreeMap<String, URLFP>(); final TreeMap<URLFP, String> urlFPToString = new TreeMap<URLFP, String>(); Set<String> list1 = Sets.newHashSet(urlList1); Set<String> list2 = Sets.newHashSet(urlList2); final Set<String> combined = Sets.union(list1, list2); Set<String> difference = Sets.difference(list1, list2); final Set<String> completedURLS = new HashSet<String>(); for (String url : combined) { URLFP fingerprint = URLUtils.getURLFPFromURL(url, true); urlToFPMap.put(url, fingerprint); urlFPToString.put(fingerprint, url); } File testInputFile1 = new File(localDir, "INPUT_LIST-" + System.currentTimeMillis()); File testInputFile2 = new File(localDir, "INPUT_LIST-" + (System.currentTimeMillis() + 1)); try { generateTestURLFile(testInputFile1, urlList1); generateTestURLFile(testInputFile2, urlList2); FileSystem localFileSystem = FileSystem.getLocal(CrawlEnvironment.getHadoopConfig()); EventLoop eventLoop = new EventLoop(); eventLoop.start(); final CrawlHistoryManager logManager = new CrawlHistoryManager(localFileSystem, new Path(remoteDir.getAbsolutePath()), localDir, eventLoop, 0); final LinkedBlockingQueue<ProxyCrawlHistoryItem> queue = new LinkedBlockingQueue<ProxyCrawlHistoryItem>(); final Semaphore initialListComplete = new Semaphore(0); logManager.startQueueLoaderThread(new CrawlQueueLoader() { @Override public void queueURL(URLFP urlfp, String url) { ProxyCrawlHistoryItem item = new ProxyCrawlHistoryItem(); item.setOriginalURL(url); queue.add(item); } @Override public void flush() { // TODO Auto-generated method stub } }); Thread queueTestThread = new Thread(new Runnable() { @Override public void run() { while (true) { try { ProxyCrawlHistoryItem item = queue.take(); if (item.getOriginalURL().length() == 0) { break; } else { System.out.println("Got:" + item.getOriginalURL()); CrawlURL urlObject = new CrawlURL(); Assert.assertTrue(!completedURLS.contains(item.getOriginalURL())); completedURLS.add(item.getOriginalURL()); urlObject.setLastAttemptResult((byte) CrawlURL.CrawlResult.SUCCESS); urlObject.setUrl(item.getOriginalURL()); urlObject.setResultCode(200); logManager.crawlComplete(urlObject); if (completedURLS.equals(combined)) { System.out.println("Hit Trigger URL. Releasing InitialListComplete Sempahore"); initialListComplete.release(1); } } } catch (InterruptedException e) { } } } }); queueTestThread.start(); logManager.loadList(testInputFile1, 0); logManager.loadList(testInputFile2, 0); System.out.println("Waiting for Initial List to Complete"); initialListComplete.acquireUninterruptibly(); System.out.println("Woke Up"); try { eventLoop.getEventThread().join(); } catch (InterruptedException e) { e.printStackTrace(); } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }