Example usage for org.apache.spark.network.util ByteUnit MiB

Introduction

In this page you can find the example usage for org.apache.spark.network.util ByteUnit MiB.

Prototype

ByteUnit MiB

To view the source code for org.apache.spark.network.util ByteUnit MiB.

Click Source Link

Usage

From source file:org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer.java

License:Apache License

public GryoSerializer(final SparkConf sparkConfiguration) {
    final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k");
    final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m");
    this.referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true);
    this.registrationRequired = sparkConfiguration.getBoolean(Constants.SPARK_KRYO_REGISTRATION_REQUIRED,
            false);//from w w  w . j  a v  a  2s.c o m
    if (bufferSizeKb >= ByteUnit.GiB.toKiB(2L)) {
        throw new IllegalArgumentException(
                "spark.kryoserializer.buffer must be less than 2048 mb, got: " + bufferSizeKb + " mb.");
    } else {
        this.bufferSize = (int) ByteUnit.KiB.toBytes(bufferSizeKb);
        if (maxBufferSizeMb >= ByteUnit.GiB.toMiB(2L)) {
            throw new IllegalArgumentException(
                    "spark.kryoserializer.buffer.max must be less than 2048 mb, got: " + maxBufferSizeMb
                            + " mb.");
        } else {
            this.maxBufferSize = (int) ByteUnit.MiB.toBytes(maxBufferSizeMb);
            //this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator");
        }
    }
    // create a GryoPool and store it in static HadoopPools
    final List<Object> ioRegistries = new ArrayList<>();
    ioRegistries.addAll(makeApacheConfiguration(sparkConfiguration).getList(IoRegistry.IO_REGISTRY,
            Collections.emptyList()));
    ioRegistries.add(SparkIoRegistry.class.getCanonicalName()
            .replace("." + SparkIoRegistry.class.getSimpleName(), "$" + SparkIoRegistry.class.getSimpleName()));
    HadoopPools.initialize(GryoPool.build()
            .poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
                    GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT))
            .ioRegistries(ioRegistries).initializeMapper(builder -> builder
                    .referenceTracking(this.referenceTracking).registrationRequired(this.registrationRequired))
            .create());
}

---