Example usage for java.util.concurrent LinkedBlockingDeque add

Introduction

In this page you can find the example usage for java.util.concurrent LinkedBlockingDeque add.

Prototype

public boolean add(E e) 

Source Link

Document

Inserts the specified element at the end of this deque unless it would violate capacity restrictions.

Usage

From source file:org.commoncrawl.service.listcrawler.DataTransferAgent.java

public static void main(String[] args) {

    Logger logger = Logger.getLogger("org.commoncrawl");
    logger.setLevel(Level.INFO);/* ww w.  j  a va 2  s .  c o  m*/
    BasicConfigurator.configure();

    Configuration conf = new Configuration();

    conf.addResource("core-site.xml");
    conf.addResource("hdfs-site.xml");

    // set a big io buffer size ... 
    conf.setInt("io.file.buffer.size", 4096 * 1024);

    final File transferLogDir = new File("/home/rana/ccprod/data/proxy_xfr_log");
    final Path hdfsCacheDataPath = new Path("crawl/proxy/cache/");
    final File shutdownFile = new File("/home/rana/ccprod/data/shutdown_xfr");

    // create a deque .. 
    final LinkedBlockingDeque<ProxyTransferItem> itemQueue = new LinkedBlockingDeque<ProxyTransferItem>();

    final EventLoop eventLoop = new EventLoop();
    eventLoop.start();

    try {

        final DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(conf);
        Thread transferThreads[] = new Thread[TRANSFER_THREADS_PER_HOST * mappingsTable.size()];
        Semaphore shutdownSemaphore = new Semaphore(0);
        int threadIndex = 0;
        for (int i = 0; i < TRANSFER_THREADS_PER_HOST; ++i) {
            int serverIdx = 0;
            for (CCBridgeServerMapping mapping : mappingsTable) {
                transferThreads[(i * mappingsTable.size()) + serverIdx++] = startTransferThread(threadIndex++,
                        mapping, shutdownFile, fs, conf, itemQueue, eventLoop, shutdownSemaphore);
            }
        }

        Thread scannerThread = new Thread(new Runnable() {

            long _lastScanId = -1;
            long _lastOutOfOrderDataDirId = -1L;

            static final int SCAN_INTERVAL_MS = 500;

            @Override
            public void run() {

                while (true) {

                    try {
                        if (shutdownFile.exists()) {
                            LOG.info("Shutdown File Detected in ScanTimer Outer Loop. Exiting Scan Thread");
                            return;
                        }

                        LOG.info("Scanning For Files based on filter. Last Known Scan Id is:" + _lastScanId);
                        FileStatus fileList[] = fs.listStatus(hdfsCacheDataPath, new PathFilter() {

                            @Override
                            public boolean accept(Path path) {
                                try {
                                    if (path.getName().startsWith("cacheData-")) {
                                        // extract file id ... 
                                        long currentFileId = Long
                                                .parseLong(path.getName().substring("cacheData-".length()));
                                        // figure out if we are going to process it ... 
                                        if (_lastScanId == -1 || currentFileId > _lastScanId) {
                                            return true;
                                        }
                                    }
                                } catch (Exception e) {
                                    LOG.error("Caught Exception Processing Path Filter:"
                                            + CCStringUtils.stringifyException(e));
                                }
                                return false;
                            }
                        });
                        LOG.info("Scan returned:" + fileList.length + " Number of Valid Files");

                        long latestFileId = 0L;
                        for (FileStatus file : fileList) {
                            // extract file id ... 
                            long currentFileId = Long
                                    .parseLong(file.getPath().getName().substring("cacheData-".length()));
                            // figure out if we are going to process it ... 
                            if (_lastScanId == -1 || currentFileId > _lastScanId) {
                                // cache max latest id ..
                                latestFileId = Math.max(latestFileId, currentFileId);
                                File logFile = hdfsCacheFileToLogFileLocation(transferLogDir, file);
                                if (logFile != null) {
                                    if (logFile.exists()) {
                                        LOG.info("Skipping:" + file.getPath().getName());
                                    } else {
                                        LOG.info("Queueing File:" + file.getPath().getName());
                                        itemQueue.add(new ProxyTransferItem(file.getPath(), logFile,
                                                file.getPath().getName()));
                                    }
                                }
                            }
                        }
                        // ok update lastest file id 
                        _lastScanId = Math.max(_lastScanId, latestFileId);

                        FileStatus outofOrderDataDirs[] = fs
                                .globStatus(new Path("crawl/proxy/dtAgentOutOfOrderTransfers/*"));

                        for (FileStatus outOfOrderDataDir : outofOrderDataDirs) {
                            long dataDirId = Long.parseLong(outOfOrderDataDir.getPath().getName());
                            if (dataDirId > _lastOutOfOrderDataDirId) {
                                FileStatus candidates[] = fs
                                        .globStatus(new Path(outOfOrderDataDir.getPath(), "part-*"));

                                for (FileStatus candidate : candidates) {
                                    File logFile = outOfOrderFileToLogFileLocation(transferLogDir,
                                            candidate.getPath());
                                    if (logFile != null) {
                                        String candidateName = candidate.getPath().getParent().getName() + "-"
                                                + candidate.getPath().getName();

                                        if (logFile.exists()) {
                                            LOG.info("Skipping OOB FILE:" + candidateName);

                                        } else {
                                            LOG.info("Queueing OOB FILE:" + candidateName);
                                            itemQueue.add(new ProxyTransferItem(candidate.getPath(), logFile,
                                                    candidateName));
                                        }
                                    }
                                }
                                _lastOutOfOrderDataDirId = dataDirId;
                            }
                        }

                        LOG.info("Finish Scan. Last Known Scan Id is now:" + _lastScanId);

                    } catch (Exception e) {
                        LOG.error(CCStringUtils.stringifyException(e));
                    }

                    try {
                        Thread.sleep(SCAN_INTERVAL_MS);
                    } catch (InterruptedException e) {
                    }
                }
            }
        });

        // start scanner thread ... 
        scannerThread.start();

        LOG.info("Waiting on Transfer Threads");
        shutdownSemaphore.acquireUninterruptibly(TRANSFER_THREADS_PER_HOST * mappingsTable.size());
        LOG.info("ALL Transfer Threads Dead.");
        // wait for scanner thread to die 
        LOG.info("Waiting for Scanner Thread to Die.");
        try {
            scannerThread.join();
        } catch (InterruptedException e) {
        }
        LOG.info("Killing Event Loop");
        eventLoop.stop();

    } catch (IOException e) {
        LOG.error(CCStringUtils.stringifyException(e));
    }

}

---