Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS.

Prototype

String DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS.

Click Source Link

Usage

From source file:org.apache.impala.util.HdfsCachingUtil.java

License:Apache License

/**
 * Waits on a cache directive to either complete or stop making progress. Progress is
 * checked by polling the HDFS caching stats every
 * DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS. We verify the request's
 * "currentBytesCached" is increasing compared to "bytesNeeded".
 * If "currentBytesCached" == "bytesNeeded" or if no progress is made for a
 * MAX_UNCHANGED_CACHING_REFRESH_INTERVALS, this function returns.
 *///w w  w. j a  v  a  2 s. c om
public static void waitForDirective(long directiveId) throws ImpalaRuntimeException {
    long bytesNeeded = 0L;
    long currentBytesCached = 0L;
    CacheDirectiveEntry cacheDir = getDirective(directiveId);
    if (cacheDir == null)
        return;

    bytesNeeded = cacheDir.getStats().getBytesNeeded();
    currentBytesCached = cacheDir.getStats().getBytesCached();
    if (LOG.isTraceEnabled()) {
        LOG.trace(String.format("Waiting on cache directive id: %d. Bytes " + "cached (%d) / needed (%d)",
                directiveId, currentBytesCached, bytesNeeded));
    }
    // All the bytes are cached, just return.
    if (bytesNeeded == currentBytesCached)
        return;

    // The refresh interval is how often HDFS will update cache directive stats. We use
    // this value to determine how frequently we should poll for changes.
    long hdfsRefreshIntervalMs = getDfs().getConf().getLong(
            DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS,
            DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT);
    Preconditions.checkState(hdfsRefreshIntervalMs > 0);

    // Loop until either MAX_UNCHANGED_CACHING_REFRESH_INTERVALS have passed with no
    // changes or all required data is cached.
    int unchangedCounter = 0;
    while (unchangedCounter < MAX_UNCHANGED_CACHING_REFRESH_INTERVALS) {
        long previousBytesCached = currentBytesCached;
        cacheDir = getDirective(directiveId);
        if (cacheDir == null)
            return;
        currentBytesCached = cacheDir.getStats().getBytesCached();
        bytesNeeded = cacheDir.getStats().getBytesNeeded();
        if (currentBytesCached == bytesNeeded) {
            if (LOG.isTraceEnabled()) {
                LOG.trace(String.format(
                        "Cache directive id: %d has completed." + "Bytes cached (%d) / needed (%d)",
                        directiveId, currentBytesCached, bytesNeeded));
            }
            return;
        }

        if (currentBytesCached == previousBytesCached) {
            ++unchangedCounter;
        } else {
            unchangedCounter = 0;
        }
        try {
            // Sleep for the refresh interval + a little bit more to ensure a full interval
            // has completed. A value of 25% the refresh interval was arbitrarily chosen.
            Thread.sleep((long) (hdfsRefreshIntervalMs * 1.25));
        } catch (InterruptedException e) {
            /* ignore */ }
    }
    LOG.warn(String.format(
            "No changes in cached bytes in: %d(ms). All data may not "
                    + "be cached. Final stats for cache directive id: %d. Bytes cached (%d)/needed " + "(%d)",
            hdfsRefreshIntervalMs * MAX_UNCHANGED_CACHING_REFRESH_INTERVALS, directiveId, currentBytesCached,
            bytesNeeded));
}

---