Example usage for org.apache.hadoop.hdfs DFSInotifyEventInputStream poll

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSInotifyEventInputStream poll.

Prototype

public EventBatch poll(long time, TimeUnit tu)
        throws IOException, InterruptedException, MissingEventsException 

Source Link

Document

Returns the next event batch in the stream, waiting up to the specified amount of time for a new batch.

Usage

From source file:alluxio.underfs.hdfs.activesync.SupportedHdfsActiveSyncProvider.java

License:Apache License

/**
 * Fetch and process events./*from  w  w w  .  j  a  va 2s  .  c o  m*/
 * @param eventStream event stream
 */
public void pollEvent(DFSInotifyEventInputStream eventStream) {
    EventBatch batch;
    LOG.debug("Polling thread starting, with timeout {} ms", mActiveUfsPollTimeoutMs);
    int count = 0;
    long start = System.currentTimeMillis();

    long behind = eventStream.getTxidsBehindEstimate();

    while (!Thread.currentThread().isInterrupted()) {
        try {
            batch = eventStream.poll(mActiveUfsPollTimeoutMs, TimeUnit.MILLISECONDS);

            if (batch != null) {
                long txId = batch.getTxid();
                count++;
                for (Event event : batch.getEvents()) {
                    processEvent(event, mUfsUriList, txId);
                }
            }
            long end = System.currentTimeMillis();
            if (end > (start + mActiveUfsSyncEventRateInterval)) {
                long currentlyBehind = eventStream.getTxidsBehindEstimate();
                LOG.info("HDFS generated {} events in {} ms, at a rate of {} rps",
                        count + currentlyBehind - behind, end - start,
                        String.format("%.2f", (count + currentlyBehind - behind) * 1000.0 / (end - start)));
                LOG.info("processed {} events in {} ms, at a rate of {} rps", count, end - start,
                        String.format("%.2f", count * 1000.0 / (end - start)));
                LOG.info("Currently TxidsBehindEstimate by {}", currentlyBehind);
                behind = currentlyBehind;
                start = end;
                count = 0;
            }
        } catch (IOException e) {
            LOG.warn("IOException occured during polling inotify {}", e);
            if (e.getCause() instanceof InterruptedException) {
                return;
            }
        } catch (MissingEventsException e) {
            LOG.warn("MissingEventException during polling {}", e);
            mEventMissed = true;
            // need to sync all syncpoints at this point
        } catch (InterruptedException e) {
            LOG.warn("InterruptedException during polling {}", e);
            return;
        }
    }
}

---

From source file:org.apache.nifi.processors.hadoop.inotify.GetHDFSEvents.java

License:Apache License

private EventBatch getEventBatch(DFSInotifyEventInputStream eventStream, long duration, TimeUnit timeUnit,
        int retries) throws IOException, InterruptedException, MissingEventsException {
    // According to the inotify API we should retry a few times if poll throws an IOException.
    // Please see org.apache.hadoop.hdfs.DFSInotifyEventInputStream#poll for documentation.
    int i = 0;//from   w  w w.j  av  a  2s.c  om
    while (true) {
        try {
            i += 1;
            return eventStream.poll(duration, timeUnit);
        } catch (IOException e) {
            if (i > retries) {
                getLogger().debug("Failed to poll for event batch. Reached max retry times.", e);
                throw e;
            } else {
                getLogger().debug("Attempt {} failed to poll for event batch. Retrying.", new Object[] { i });
            }
        }
    }
}

---