Example usage for java.util.concurrent Semaphore release

List of usage examples for java.util.concurrent Semaphore release

Introduction

In this page you can find the example usage for java.util.concurrent Semaphore release.

Prototype

public void release(int permits) 

Source Link

Document

Releases the given number of permits, returning them to the semaphore.

Usage

From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java

private static void launchInTestMode() {

    File baseTestDir = new File("/tmp/logManagerTest");
    FileUtils.recursivelyDeleteFile(baseTestDir);
    baseTestDir.mkdir();/*from   www .ja  v a 2 s  .  com*/
    File remoteDir = new File(baseTestDir, "remote");
    File localDir = new File(baseTestDir, "local");
    remoteDir.mkdir();
    localDir.mkdir();

    final TreeMap<String, URLFP> urlToFPMap = new TreeMap<String, URLFP>();
    final TreeMap<URLFP, String> urlFPToString = new TreeMap<URLFP, String>();

    Set<String> list1 = Sets.newHashSet(urlList1);
    Set<String> list2 = Sets.newHashSet(urlList2);
    final Set<String> combined = Sets.union(list1, list2);
    Set<String> difference = Sets.difference(list1, list2);
    final Set<String> completedURLS = new HashSet<String>();
    for (String url : combined) {
        URLFP fingerprint = URLUtils.getURLFPFromURL(url, true);
        urlToFPMap.put(url, fingerprint);
        urlFPToString.put(fingerprint, url);
    }

    File testInputFile1 = new File(localDir, "INPUT_LIST-" + System.currentTimeMillis());
    File testInputFile2 = new File(localDir, "INPUT_LIST-" + (System.currentTimeMillis() + 1));

    try {

        generateTestURLFile(testInputFile1, urlList1);
        generateTestURLFile(testInputFile2, urlList2);

        FileSystem localFileSystem = FileSystem.getLocal(CrawlEnvironment.getHadoopConfig());

        EventLoop eventLoop = new EventLoop();
        eventLoop.start();

        final CrawlHistoryManager logManager = new CrawlHistoryManager(localFileSystem,
                new Path(remoteDir.getAbsolutePath()), localDir, eventLoop, 0);

        final LinkedBlockingQueue<ProxyCrawlHistoryItem> queue = new LinkedBlockingQueue<ProxyCrawlHistoryItem>();

        final Semaphore initialListComplete = new Semaphore(0);

        logManager.startQueueLoaderThread(new CrawlQueueLoader() {

            @Override
            public void queueURL(URLFP urlfp, String url) {
                ProxyCrawlHistoryItem item = new ProxyCrawlHistoryItem();
                item.setOriginalURL(url);
                queue.add(item);
            }

            @Override
            public void flush() {
                // TODO Auto-generated method stub

            }
        });

        Thread queueTestThread = new Thread(new Runnable() {

            @Override
            public void run() {
                while (true) {
                    try {
                        ProxyCrawlHistoryItem item = queue.take();

                        if (item.getOriginalURL().length() == 0) {
                            break;
                        } else {

                            System.out.println("Got:" + item.getOriginalURL());

                            CrawlURL urlObject = new CrawlURL();

                            Assert.assertTrue(!completedURLS.contains(item.getOriginalURL()));
                            completedURLS.add(item.getOriginalURL());

                            urlObject.setLastAttemptResult((byte) CrawlURL.CrawlResult.SUCCESS);
                            urlObject.setUrl(item.getOriginalURL());
                            urlObject.setResultCode(200);

                            logManager.crawlComplete(urlObject);

                            if (completedURLS.equals(combined)) {
                                System.out.println("Hit Trigger URL. Releasing InitialListComplete Sempahore");
                                initialListComplete.release(1);
                            }
                        }

                    } catch (InterruptedException e) {
                    }
                }
            }

        });

        queueTestThread.start();

        logManager.loadList(testInputFile1, 0);
        logManager.loadList(testInputFile2, 0);
        System.out.println("Waiting for Initial List to Complete");
        initialListComplete.acquireUninterruptibly();
        System.out.println("Woke Up");

        try {
            eventLoop.getEventThread().join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}