Example usage for io.netty.util.internal PlatformDependent maxDirectMemory

Introduction

In this page you can find the example usage for io.netty.util.internal PlatformDependent maxDirectMemory.

Prototype

public static long maxDirectMemory() 

Source Link

Document

Returns the maximum memory reserved for direct buffer allocation.

Usage

From source file:org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.java

License:Apache License

@Override
public void initialize(ServerConfiguration conf, LedgerManager ledgerManager,
        LedgerDirsManager ledgerDirsManager, LedgerDirsManager indexDirsManager, StateManager stateManager,
        CheckpointSource checkpointSource, Checkpointer checkpointer, StatsLogger statsLogger,
        ByteBufAllocator allocator) throws IOException {
    long writeCacheMaxSize = getLongVariableOrDefault(conf, WRITE_CACHE_MAX_SIZE_MB,
            DEFAULT_WRITE_CACHE_MAX_SIZE_MB) * MB;
    long readCacheMaxSize = getLongVariableOrDefault(conf, READ_AHEAD_CACHE_MAX_SIZE_MB,
            DEFAULT_READ_CACHE_MAX_SIZE_MB) * MB;

    this.allocator = allocator;
    this.numberOfDirs = ledgerDirsManager.getAllLedgerDirs().size();

    log.info("Started Db Ledger Storage");
    log.info(" - Number of directories: {}", numberOfDirs);
    log.info(" - Write cache size: {} MB", writeCacheMaxSize / MB);
    log.info(" - Read Cache: {} MB", readCacheMaxSize / MB);

    if (readCacheMaxSize + writeCacheMaxSize > PlatformDependent.maxDirectMemory()) {
        throw new IOException("Read and write cache sizes exceed the configured max direct memory size");
    }/*  w ww .  ja  v a2 s  .  co m*/

    long perDirectoryWriteCacheSize = writeCacheMaxSize / numberOfDirs;
    long perDirectoryReadCacheSize = readCacheMaxSize / numberOfDirs;

    gcExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollector"));

    ledgerStorageList = Lists.newArrayList();
    for (File ledgerDir : ledgerDirsManager.getAllLedgerDirs()) {
        // Create a ledger dirs manager for the single directory
        File[] dirs = new File[1];
        // Remove the `/current` suffix which will be appended again by LedgersDirManager
        dirs[0] = ledgerDir.getParentFile();
        LedgerDirsManager ldm = new LedgerDirsManager(conf, dirs, ledgerDirsManager.getDiskChecker(),
                statsLogger);
        ledgerStorageList.add(newSingleDirectoryDbLedgerStorage(conf, ledgerManager, ldm, indexDirsManager,
                stateManager, checkpointSource, checkpointer, statsLogger, gcExecutor,
                perDirectoryWriteCacheSize, perDirectoryReadCacheSize));
    }

    this.stats = new DbLedgerStorageStats(statsLogger,
            () -> ledgerStorageList.stream().mapToLong(SingleDirectoryDbLedgerStorage::getWriteCacheSize).sum(),
            () -> ledgerStorageList.stream().mapToLong(SingleDirectoryDbLedgerStorage::getWriteCacheCount)
                    .sum(),
            () -> ledgerStorageList.stream().mapToLong(SingleDirectoryDbLedgerStorage::getReadCacheSize).sum(),
            () -> ledgerStorageList.stream().mapToLong(SingleDirectoryDbLedgerStorage::getReadCacheCount)
                    .sum());
}

---

From source file:org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageRocksDB.java

License:Apache License

public KeyValueStorageRocksDB(String path, DbConfigType dbConfigType, ServerConfiguration conf,
        boolean readOnly) throws IOException {
    try {//  w w w.  j  ava 2 s.  c om
        RocksDB.loadLibrary();
    } catch (Throwable t) {
        throw new IOException("Failed to load RocksDB JNI library", t);
    }

    this.optionSync = new WriteOptions();
    this.optionDontSync = new WriteOptions();
    this.optionCache = new ReadOptions();
    this.optionDontCache = new ReadOptions();
    this.emptyBatch = new WriteBatch();

    try (Options options = new Options()) {
        options.setCreateIfMissing(true);

        if (dbConfigType == DbConfigType.Huge) {
            // Set default RocksDB block-cache size to 10% of direct mem, unless override
            long defaultRocksDBBlockCacheSizeBytes = PlatformDependent.maxDirectMemory() / 10;
            long blockCacheSize = DbLedgerStorage.getLongVariableOrDefault(conf, ROCKSDB_BLOCK_CACHE_SIZE,
                    defaultRocksDBBlockCacheSizeBytes);

            long writeBufferSizeMB = conf.getInt(ROCKSDB_WRITE_BUFFER_SIZE_MB, 64);
            long sstSizeMB = conf.getInt(ROCKSDB_SST_SIZE_MB, 64);
            int numLevels = conf.getInt(ROCKSDB_NUM_LEVELS, -1);
            int numFilesInLevel0 = conf.getInt(ROCKSDB_NUM_FILES_IN_LEVEL0, 4);
            long maxSizeInLevel1MB = conf.getLong(ROCKSDB_MAX_SIZE_IN_LEVEL1_MB, 256);
            int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024);
            int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10);
            boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true);

            if (lz4CompressionEnabled) {
                options.setCompressionType(CompressionType.LZ4_COMPRESSION);
            }
            options.setWriteBufferSize(writeBufferSizeMB * 1024 * 1024);
            options.setMaxWriteBufferNumber(4);
            if (numLevels > 0) {
                options.setNumLevels(numLevels);
            }
            options.setLevelZeroFileNumCompactionTrigger(numFilesInLevel0);
            options.setMaxBytesForLevelBase(maxSizeInLevel1MB * 1024 * 1024);
            options.setMaxBackgroundCompactions(16);
            options.setMaxBackgroundFlushes(16);
            options.setIncreaseParallelism(32);
            options.setMaxTotalWalSize(512 * 1024 * 1024);
            options.setMaxOpenFiles(-1);
            options.setTargetFileSizeBase(sstSizeMB * 1024 * 1024);
            options.setDeleteObsoleteFilesPeriodMicros(TimeUnit.HOURS.toMicros(1));

            BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
            tableOptions.setBlockSize(blockSize);
            tableOptions.setBlockCacheSize(blockCacheSize);
            tableOptions.setFormatVersion(2);
            tableOptions.setChecksumType(ChecksumType.kxxHash);
            if (bloomFilterBitsPerKey > 0) {
                tableOptions.setFilter(new BloomFilter(bloomFilterBitsPerKey, false));
            }

            // Options best suited for HDDs
            tableOptions.setCacheIndexAndFilterBlocks(true);
            options.setLevelCompactionDynamicLevelBytes(true);

            options.setTableFormatConfig(tableOptions);
        }

        // Configure log level
        String logLevel = conf.getString(ROCKSDB_LOG_LEVEL, "info");
        switch (logLevel) {
        case "debug":
            options.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL);
            break;
        case "info":
            options.setInfoLogLevel(InfoLogLevel.INFO_LEVEL);
            break;
        case "warn":
            options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
            break;
        case "error":
            options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
            break;
        default:
            log.warn("Unrecognized RockDB log level: {}", logLevel);
        }

        // Keep log files for 1month
        options.setKeepLogFileNum(30);
        options.setLogFileTimeToRoll(TimeUnit.DAYS.toSeconds(1));

        try {
            if (readOnly) {
                db = RocksDB.openReadOnly(options, path);
            } else {
                db = RocksDB.open(options, path);
            }
        } catch (RocksDBException e) {
            throw new IOException("Error open RocksDB database", e);
        }
    }

    optionSync.setSync(true);
    optionDontSync.setSync(false);

    optionCache.setFillCache(true);
    optionDontCache.setFillCache(false);
}

---

From source file:org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider.java

License:Apache License

@Override
public void start(Configuration conf) {
    boolean httpEnabled = conf.getBoolean(PROMETHEUS_STATS_HTTP_ENABLE, DEFAULT_PROMETHEUS_STATS_HTTP_ENABLE);
    boolean bkHttpServerEnabled = conf.getBoolean("httpServerEnabled", false);
    // only start its own http server when prometheus http is enabled and bk http server is not enabled.
    if (httpEnabled && !bkHttpServerEnabled) {
        int httpPort = conf.getInt(PROMETHEUS_STATS_HTTP_PORT, DEFAULT_PROMETHEUS_STATS_HTTP_PORT);
        InetSocketAddress httpEndpoint = InetSocketAddress.createUnresolved("0.0.0.0", httpPort);
        this.server = new Server(httpEndpoint);
        ServletContextHandler context = new ServletContextHandler();
        context.setContextPath("/");
        server.setHandler(context);/*from   w  w w . j  a  v a  2  s. com*/

        context.addServlet(new ServletHolder(new PrometheusServlet(this)), "/metrics");

        try {
            server.start();
            log.info("Started Prometheus stats endpoint at {}", httpEndpoint);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    // Include standard JVM stats
    registerMetrics(new StandardExports());
    registerMetrics(new MemoryPoolsExports());
    registerMetrics(new GarbageCollectorExports());
    registerMetrics(new ThreadExports());

    // Add direct memory allocated through unsafe
    registerMetrics(Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() {
        @Override
        public double get() {
            return directMemoryUsage != null ? directMemoryUsage.longValue() : Double.NaN;
        }
    }));

    registerMetrics(Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() {
        @Override
        public double get() {
            return PlatformDependent.maxDirectMemory();
        }
    }));

    executor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("metrics"));

    int latencyRolloverSeconds = conf.getInt(PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS,
            DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS);

    executor.scheduleAtFixedRate(() -> {
        rotateLatencyCollection();
    }, 1, latencyRolloverSeconds, TimeUnit.SECONDS);

}

---

From source file:org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.java

License:Apache License

public static SystemResourceUsage getSystemResourceUsage(final BrokerHostUsage brokerHostUsage)
        throws IOException {
    SystemResourceUsage systemResourceUsage = brokerHostUsage.getBrokerHostUsage();

    // Override System memory usage and limit with JVM heap usage and limit
    long maxHeapMemoryInBytes = Runtime.getRuntime().maxMemory();
    long memoryUsageInBytes = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
    systemResourceUsage.memory.usage = (double) memoryUsageInBytes / MIBI;
    systemResourceUsage.memory.limit = (double) maxHeapMemoryInBytes / MIBI;

    // Collect JVM direct memory
    systemResourceUsage.directMemory.usage = (double) (getJvmDirectMemoryUsed() / MIBI);
    systemResourceUsage.directMemory.limit = (double) (PlatformDependent.maxDirectMemory() / MIBI);

    return systemResourceUsage;
}

---

From source file:org.apache.pulsar.common.stats.JvmMetrics.java

License:Apache License

public List<Metrics> generate() {

    Metrics m = createMetrics();/*from   ww w  . ja v a 2s  .  c  om*/

    Runtime r = Runtime.getRuntime();

    m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
    m.put("jvm_max_memory", r.maxMemory());
    m.put("jvm_total_memory", r.totalMemory());

    m.put("jvm_direct_memory_used", getJvmDirectMemoryUsed());
    m.put("jvm_max_direct_memory", PlatformDependent.maxDirectMemory());
    m.put("jvm_thread_cnt", getThreadCount());

    this.gcLogger.logMetrics(m);

    long totalAllocated = 0;
    long totalUsed = 0;

    for (PoolArenaMetric arena : PooledByteBufAllocator.DEFAULT.directArenas()) {
        for (PoolChunkListMetric list : arena.chunkLists()) {
            for (PoolChunkMetric chunk : list) {
                int size = chunk.chunkSize();
                int used = size - chunk.freeBytes();

                totalAllocated += size;
                totalUsed += used;
            }
        }
    }

    m.put(this.componentName + "_default_pool_allocated", totalAllocated);
    m.put(this.componentName + "_default_pool_used", totalUsed);

    return Lists.newArrayList(m);
}

---

From source file:org.apache.tajo.plan.function.stream.BufferPool.java

License:Apache License

public static long maxDirectMemory() {
    return PlatformDependent.maxDirectMemory();
}

---