List of usage examples for org.apache.hadoop.util StringUtils byteDesc
public static String byteDesc(long len)
From source file:com.cloudera.crunchts.CrunchTSApp.java
License:Apache License
/** * Gives a report for a time series bucket. * * @exception IOException if the tsbucket does not exist. *//*from w w w . j ava2 s . com*/ public void report(String tsbFilePath) throws IOException { System.out.println("Time-Series-Bucket report (TSBr) is comming soon ..."); System.out.println("> path: " + tsbFilePath); if (fs instanceof DistributedFileSystem) { DistributedFileSystem dfs = (DistributedFileSystem) fs; long capacity = dfs.getDiskStatus().getCapacity(); long used = dfs.getDiskStatus().getDfsUsed(); long remaining = dfs.getDiskStatus().getRemaining(); long presentCapacity = used + remaining; System.out.println("FS Configured Capacity: " + capacity + " (" + StringUtils.byteDesc(capacity) + ")"); System.out.println( "FS Present Capacity: " + presentCapacity + " (" + StringUtils.byteDesc(presentCapacity) + ")"); System.out.println("DFS Remaining: " + remaining + " (" + StringUtils.byteDesc(remaining) + ")"); System.out.println("DFS Used: " + used + " (" + StringUtils.byteDesc(used) + ")"); System.out.println( "DFS Used%: " + StringUtils.limitDecimalTo2(((1.0 * used) / presentCapacity) * 100) + "%"); } }
From source file:com.inclouds.hbase.rowcache.RowCache.java
License:Open Source License
/** * Start co-processor - cache.//w w w.j a v a2 s . c o m * * @param cfg * the cfg * @throws IOException * Signals that an I/O exception has occurred. */ public void start(Configuration cfg) throws IOException { // Get all config from Configuration object // Start - load cache this.config = cfg; synchronized (RowCache.class) { if (rowCache != null) return; final CacheConfiguration ccfg = ConfigHelper.getCacheConfiguration(cfg); // set cache name ccfg.setCacheName("row-cache"); long maxMemory = cfg.getLong(ROWCACHE_MAXMEMORY, DEFAULT_MAX_MEMORY); ccfg.setMaxMemory(maxMemory); LOG.info("[row-cache] Setting max memory to " + maxMemory); long maxItems = cfg.getLong(ROWCACHE_MAXITEMS, DEFAULT_MAXITEMS); if (maxItems > Integer.MAX_VALUE - 1) { maxItems = Integer.MAX_VALUE - 1; LOG.warn("[row-cache] Max items is too large " + maxItems); } else { LOG.info("[row-cache] Setting max items to " + maxItems); } LOG.info("[row-cache] Direct memory buffer size set to " + StringUtils.byteDesc(RowCache.ioBufferSize)); ccfg.setBucketNumber((int) maxItems); String codecName = cfg.get(ROWCACHE_COMPRESSION, DEFAULT_COMPRESSION); ccfg.setCodecType(CodecType.valueOf(codecName.toUpperCase())); LOG.info("[row-cache] compression codec=" + codecName); isPersistentCache = Boolean.parseBoolean(cfg.get(ROWCACHE_PERSISTENT, DEFAULT_PERSISTENT)); LOG.info("[row-cache] persistent=" + isPersistentCache); String[] dataRoots = getDataRoots(cfg.get(ROWCACHE_CACHE_DATA_ROOTS)); if (isPersistentCache && dataRoots == null) { dataRoots = getHDFSRoots(cfg); if (dataRoots == null) { LOG.warn("Data roots are not defined for Row Cache. Set persistent mode to false."); isPersistentCache = false; } } // TODO - compression CacheManager manager = CacheManager.getInstance(); try { if (isPersistentCache) { RawFSConfiguration storeConfig = new RawFSConfiguration(); storeConfig.setDiskStoreImplementation(RawFSStore.class); storeConfig.setStoreName(ccfg.getCacheName()); storeConfig.setDbDataStoreRoots(dataRoots); storeConfig.setPersistenceMode(PersistenceMode.ONDEMAND); storeConfig.setDbSnapshotInterval(15); ccfg.setDataStoreConfiguration(storeConfig); // Load cache data rowCache = manager.getCache(ccfg, null); } else { rowCache = manager.getCache(ccfg, new ProgressListener() { @Override public void canceled() { LOG.info("Canceled"); } @Override public void error(Throwable t, boolean aborted) { LOG.error("Aborted=" + aborted, t); } @Override public void finished() { LOG.info("Finished loading cache"); } @Override public void progress(long done, long total) { LOG.info("Loaded " + done + " out of " + total); } @Override public void started() { LOG.info("Started loading scan cache data from " + ccfg.getDiskStoreConfiguration().getDbDataStoreRoots()); } }); } } catch (Throwable ex) { throw new IOException(ex); } LOG.info("[row-cache] coprocessor started "); RowCache.instance = this; Runnable r = new Runnable() { public void run() { LOG.info("[row-cache] Stats thread started. "); while (true) { try { Thread.sleep(STATS_INTERVAL); } catch (InterruptedException e) { } long lastR = lastRequests; long lastH = lastHits; long requests = rowCache.getTotalRequestCount(); long hits = rowCache.getHitCount(); if (requests != lastRequests) { // Log only if new data LOG.info("[L1-OFFHEAP]: accesses=" + requests + " hits=" + hits + " hitRatio=" + ((requests == 0) ? "0.00" : StringUtils.formatPercent((double) hits / requests, 2) + "%" + " Last period: accesses=" + (requests - lastR) + " hits=" + (hits - lastH) + " hitRatio=" + (((requests - lastR) == 0) ? "0.00" : StringUtils.formatPercent( (double) (hits - lastH) / (requests - lastR), 2))) + "%" + " maxMemory=" + StringUtils.byteDesc(rowCache.getMemoryLimit()) + " allocatedMemory=" + StringUtils.byteDesc(rowCache.getAllocatedMemorySize()) + " freeMemory=" + StringUtils .byteDesc(rowCache.getMemoryLimit() - rowCache.getAllocatedMemorySize()) + " totalItems=" + rowCache.size() + " evicted=" + rowCache.getEvictedCount()); lastRequests = requests; lastHits = hits; } } } }; statThread = new Thread(r, "BigBaseRowCache.StatisticsThread"); statThread.start(); // Register shutdown hook registerShutdownHook(); } }
From source file:com.inclouds.hbase.rowcache.RowCache.java
License:Open Source License
private void registerShutdownHook() { if (isPersistentCache == false) return;//from w w w . j a va2 s .c o m Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { LOG.info("Shutting down row-cache, saving data started ..."); long startTime = System.currentTimeMillis(); long totalRows = rowCache.size(); long totalSize = rowCache.getAllocatedMemorySize(); try { rowCache.save(); } catch (KodaException e) { LOG.error("Failed to save row-cache", e); return; } LOG.info("Saved " + StringUtils.byteDesc(totalSize) + " Total Rows:" + totalRows + " in " + (System.currentTimeMillis() - startTime) + " ms"); } }); LOG.info("[row-cache] Registered shutdown hook"); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCache.java
License:Open Source License
/** * Log stats.// w w w . ja v a 2s . c o m */ protected void logStats() { // Log size long totalSize = getCurrentSize(); long freeSize = getMaxSize() - totalSize; OffHeapBlockCache.LOG.info("[BLOCK CACHE]: " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(getMaxSize()) + ", " + "blocks=" + size() + ", " + "accesses=" + stats.getRequestCount() + ", " + "hits=" + stats.getHitCount() + ", " + "hitRatio=" + (stats.getRequestCount() > 0 ? StringUtils.formatPercent(stats.getHitRatio(), 2) : "0.00") + "%, " + "cachingAccesses=" + stats.getRequestCachingCount() + ", " + "cachingHits=" + stats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (stats.getRequestCachingCount() > 0 ? StringUtils.formatPercent(stats.getHitCachingRatio(), 2) : "0.00") + "%, " + "evicted=" + getEvictedCount()); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCache.java
License:Open Source License
protected void logStatsOffHeap() { // Log size/* www . j a v a 2 s.c om*/ long totalSize = offHeapCache.getTotalAllocatedMemorySize(); long maxSize = offHeapCache.getMemoryLimit(); long freeSize = maxSize - totalSize; OffHeapBlockCache.LOG.info("[L2-OFFHEAP] : " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(maxSize) + ", " + "blocks=" + offHeapCache.size() + ", " + "accesses=" + offHeapStats.getRequestCount() + ", " + "hits=" + offHeapStats.getHitCount() + ", " + "hitRatio=" + (offHeapStats.getRequestCount() > 0 ? StringUtils.formatPercent(offHeapStats.getHitRatio(), 2) : "0.00") + "%, " + "cachingAccesses=" + offHeapStats.getRequestCachingCount() + ", " + "cachingHits=" + offHeapStats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (offHeapStats.getRequestCachingCount() > 0 ? StringUtils.formatPercent(offHeapStats.getHitCachingRatio(), 2) : "0.00") + "%, " + "evicted=" + offHeapCache.getEvictedCount()); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCache.java
License:Open Source License
protected void logStatsOffHeapExt() { // Log size//from w ww . j a va2 s. c o m long totalSize = extStorageCache.getAllocatedMemorySize(); long maxSize = extStorageCache.getMemoryLimit(); long freeSize = maxSize - totalSize; OffHeapBlockCache.LOG.info("[L3-OFFHEAP] : " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(maxSize) + ", " + "refs=" + extStorageCache.size() + ", " + "accesses=" + extRefStats.getRequestCount() + ", " + "hits=" + extRefStats.getHitCount() + ", " + "hitRatio=" + (extRefStats.getRequestCount() > 0 ? StringUtils.formatPercent(extRefStats.getHitRatio(), 2) : "0.00") + "%, " + "cachingAccesses=" + extRefStats.getRequestCachingCount() + ", " + "cachingHits=" + extRefStats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (extRefStats.getRequestCachingCount() > 0 ? StringUtils.formatPercent(offHeapStats.getHitCachingRatio(), 2) : "0.00") + "%, " + "evicted=" + extStorageCache.getEvictedCount()); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCache.java
License:Open Source License
protected void logStatsOnHeap() { if (onHeapEnabled() == false) return;//w w w. j a v a 2 s .c om // Log size long totalSize = onHeapCache.getCurrentSize(); long maxSize = onHeapCache.getMaxSize(); long freeSize = maxSize - totalSize; OnHeapBlockCache.LOG.info("[L2-HEAP] : " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(maxSize) + ", " + "blocks=" + onHeapCache.size() + ", " + "accesses=" + onHeapStats.getRequestCount() + ", " + "hits=" + onHeapStats.getHitCount() + ", " + "hitRatio=" + (onHeapStats.getRequestCount() > 0 ? StringUtils.formatPercent(onHeapStats.getHitRatio(), 2) : "0.00") + "%, " + "cachingAccesses=" + onHeapStats.getRequestCachingCount() + ", " + "cachingHits=" + onHeapStats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (onHeapStats.getRequestCachingCount() > 0 ? StringUtils.formatPercent(onHeapStats.getHitCachingRatio(), 2) : "0.00") + "%, " + "evicted=" + onHeapCache.getEvictedCount()); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCache.java
License:Open Source License
protected void logStatsExternal() { if (storage == null) return;//from www .jav a 2 s.c o m // Log size long totalSize = storage.size(); long maxSize = storage.getMaxStorageSize(); long freeSize = maxSize - totalSize; OffHeapBlockCache.LOG.info("[L3-DISK] : " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(maxSize) + ", " + "accesses=" + extStats.getRequestCount() + ", " + "hits=" + extStats.getHitCount() + ", " + "hitRatio=" + (extStats.getRequestCount() > 0 ? StringUtils.formatPercent(extStats.getHitRatio(), 2) : "0.00") + "%, " + "cachingAccesses=" + extStats.getRequestCachingCount() + ", " + "cachingHits=" + extStats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (extStats.getRequestCachingCount() > 0 ? StringUtils.formatPercent(extStats.getHitCachingRatio(), 2) : "0.00") + "%, "); // "\nFATAL READS="+fatalExternalReads.get()); }
From source file:com.koda.integ.hbase.blockcache.OffHeapBlockCacheOld.java
License:Open Source License
/** * Log stats./*from w w w.ja v a 2 s .c om*/ */ public void logStats() { if (!LOG.isDebugEnabled()) return; // Log size long totalSize = getCurrentSize(); long freeSize = maxSize - totalSize; OffHeapBlockCacheOld.LOG.debug("LRU Stats: " + "total=" + StringUtils.byteDesc(totalSize) + ", " + "free=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(this.maxSize) + ", " + "blocks=" + size() + ", " + "accesses=" + stats.getRequestCount() + ", " + "hits=" + stats.getHitCount() + ", " + "hitRatio=" + StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, " + "cachingAccesses=" + stats.getRequestCachingCount() + ", " + "cachingHits=" + stats.getHitCachingCount() + ", " + "cachingHitsRatio=" + StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + "%, " + "evicted=" + getEvictedCount()); }
From source file:com.koda.integ.hbase.blockcache.OnHeapBlockCache.java
License:Open Source License
/** * Eviction method.//from ww w. ja va2 s.c o m */ void evict() { // Ensure only one eviction at a time if (!evictionLock.tryLock()) return; try { evictionInProgress = true; long currentSize = this.size.get(); long bytesToFree = currentSize - minSize(); if (LOG.isDebugEnabled()) { LOG.debug("Block cache LRU eviction started; Attempting to free " + StringUtils.byteDesc(bytesToFree) + " of total=" + StringUtils.byteDesc(currentSize)); } if (bytesToFree <= 0) return; // Instantiate priority buckets BlockBucket bucketSingle = new BlockBucket(bytesToFree, blockSize, singleSize()); BlockBucket bucketMulti = new BlockBucket(bytesToFree, blockSize, multiSize()); BlockBucket bucketMemory = new BlockBucket(bytesToFree, blockSize, memorySize()); // Scan entire map putting into appropriate buckets for (CachedBlock cachedBlock : map.values()) { switch (cachedBlock.getPriority()) { case SINGLE: { bucketSingle.add(cachedBlock); break; } case MULTI: { bucketMulti.add(cachedBlock); break; } case MEMORY: { bucketMemory.add(cachedBlock); break; } } } PriorityQueue<BlockBucket> bucketQueue = new PriorityQueue<BlockBucket>(3); bucketQueue.add(bucketSingle); bucketQueue.add(bucketMulti); bucketQueue.add(bucketMemory); int remainingBuckets = 3; long bytesFreed = 0; BlockBucket bucket; while ((bucket = bucketQueue.poll()) != null) { long overflow = bucket.overflow(); if (overflow > 0) { long bucketBytesToFree = Math.min(overflow, (bytesToFree - bytesFreed) / remainingBuckets); bytesFreed += bucket.free(bucketBytesToFree); } remainingBuckets--; } if (LOG.isDebugEnabled()) { long single = bucketSingle.totalSize(); long multi = bucketMulti.totalSize(); long memory = bucketMemory.totalSize(); LOG.debug("Block cache LRU eviction completed; " + "freed=" + StringUtils.byteDesc(bytesFreed) + ", " + "total=" + StringUtils.byteDesc(this.size.get()) + ", " + "single=" + StringUtils.byteDesc(single) + ", " + "multi=" + StringUtils.byteDesc(multi) + ", " + "memory=" + StringUtils.byteDesc(memory)); } } finally { stats.evict(); evictionInProgress = false; evictionLock.unlock(); } }