Example usage for org.apache.spark.network TransportContext TransportContext

Introduction

In this page you can find the example usage for org.apache.spark.network TransportContext TransportContext.

Prototype

public TransportContext(TransportConf conf, RpcHandler rpcHandler) 

Source Link

Usage

From source file:org.apache.carbondata.spark.dictionary.client.SecureDictionaryClient.java

License:Apache License

/**
 * start dictionary client//from  w w  w . j  a v  a2  s . c om
 *
 * @param address
 * @param port
 */
@Override
public void startClient(String secretKey, String address, int port, boolean encryptSecureServer) {
    LOGGER.info("Starting client on " + address + " " + port);
    long start = System.currentTimeMillis();

    SecurityManager securityMgr;
    SparkConf conf = new SparkConf().setAppName("Carbon Dictionary Client");

    conf.set("spark.authenticate", "true");

    if (null != secretKey) {
        conf.set("spark.authenticate.secret", secretKey);
    }

    if (encryptSecureServer) {
        conf.set("spark.authenticate.enableSaslEncryption", "true");
    }

    TransportConf transportConf = SparkTransportConf.fromSparkConf(conf, "Carbon Dictionary Client", 0);
    if (null != secretKey) {
        securityMgr = new SecurityManager(conf,
                scala.Option.apply(secretKey.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))));
    } else {
        securityMgr = new SecurityManager(conf, null);
    }

    TransportContext context = new TransportContext(transportConf, dictionaryClientHandler);
    clientFactory = context.createClientFactory(Lists.<TransportClientBootstrap>newArrayList(
            new SaslClientBootstrap(transportConf, "Carbon Dictionary Client", securityMgr)));

    try {
        client = clientFactory.createClient(address, port);
    } catch (Exception e) {
        LOGGER.error("Dictionary Client Failed to bind to port:", e);
    }
    LOGGER.info("Dictionary client Started, Total time spent : " + (System.currentTimeMillis() - start));
}

---

From source file:org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer.java

License:Apache License

/**
 * Binds dictionary server to an available port.
 *
 *///  w  w w  . j a v  a2s  . com
@Override
public void bindToPort() {
    long start = System.currentTimeMillis();
    // Configure the server.
    int i = 0;
    while (i < 10) {
        int newPort = port + i;
        try {
            SecurityManager securityManager;
            SparkConf conf = this.conf.clone();
            conf.setAppName("Carbon Dictionary Server");

            // As spark.network.sasl.serverAlwaysEncrypt is not exposed parameter
            // set it explicitly so that Dictionary Server and Client communication
            // occurs encrypted. The below parameter can be removed once spark Documents it.
            // conf.set("spark.network.sasl.serverAlwaysEncrypt", "true");
            conf.set("spark.authenticate.enableSaslEncryption", "true");

            if (conf.get("spark.authenticate.enableSaslEncryption", "false").equalsIgnoreCase("true")) {
                setEncryptSecureServer(true);
            } else {
                setEncryptSecureServer(false);
            }

            TransportConf transportConf = SparkTransportConf.fromSparkConf(conf, "Carbon Dictionary Server", 0);
            securityManager = new SecurityManager(conf, Some.<byte[]>empty());
            secretKey = securityManager.getSecretKey();
            TransportContext context = new TransportContext(transportConf, secureDictionaryServerHandler);
            TransportServerBootstrap bootstrap = new SaslServerBootstrap(transportConf, securityManager);
            String host = findLocalIpAddress(LOGGER);
            //iteratively listening to newports
            context.createServer(host, newPort, Lists.<TransportServerBootstrap>newArrayList(bootstrap));
            LOGGER.info("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start)
                    + " Listening on port " + newPort);
            this.port = newPort;
            this.host = host;
            break;
        } catch (Exception e) {
            LOGGER.error("Dictionary Server Failed to bind to port: " + newPort, e);
            if (i == 9) {
                throw new RuntimeException("Dictionary Server Could not bind to any port");
            }
        }
        i++;
    }
}

---

From source file:org.apache.sysml.runtime.controlprogram.paramserv.rpc.PSRpcFactory.java

License:Apache License

private static TransportContext createTransportContext(SparkConf conf, LocalParamServer ps) {
    TransportConf tc = SparkTransportConf.fromSparkConf(conf, MODULE_NAME, 0);
    PSRpcHandler handler = new PSRpcHandler(ps);
    return new TransportContext(tc, handler);
}

---