Example usage for org.apache.hadoop.net NetUtils getFreeSocketPort

List of usage examples for org.apache.hadoop.net NetUtils getFreeSocketPort

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.net.*
  5. NetUtils
  6. getFreeSocketPort

Introduction

In this page you can find the example usage for org.apache.hadoop.net NetUtils getFreeSocketPort.

Prototype

public static int getFreeSocketPort()

Source Link

Document

Return a free port number.

Usage

From source file:com.inforefiner.hdata.SubmitClient.java

License:Apache License

/**
 * Parse command line options/*from  w  w w.  ja  v a  2  s.  c  o  m*/
 *
 * @param args Parsed command line options
 * @return Whether the init was successful to run the client
 * @throws ParseException
 */
public boolean init(String[] args) throws ParseException {

    CommandLine cliParser = new GnuParser().parse(opts, args);

    if (args.length == 0) {
        throw new IllegalArgumentException("No args specified for client to initialize");
    }

    if (cliParser.hasOption("log_properties")) {
        String log4jPath = cliParser.getOptionValue("log_properties");
        try {
            Log4jPropertyHelper.updateLog4jConfiguration(SubmitClient.class, log4jPath);
        } catch (Exception e) {
            LOG.warn("Can not set up custom log4j properties. " + e);
        }
    }

    if (cliParser.hasOption("help")) {
        printUsage();
        return false;
    }

    if (cliParser.hasOption("debug")) {
        debugFlag = true;

    }

    if (cliParser.hasOption("keep_containers_across_application_attempts")) {
        LOG.info("keep_containers_across_application_attempts");
        keepContainers = true;
    }

    appName = cliParser.getOptionValue("appname", "hdata-cloud-job-" + System.currentTimeMillis());
    amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
    amQueue = cliParser.getOptionValue("queue", "default");
    amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "10"));
    amVCores = Integer.parseInt(cliParser.getOptionValue("master_vcores", "1"));

    if (amMemory < 0) {
        throw new IllegalArgumentException(
                "Invalid memory specified for application master, exiting." + " Specified memory=" + amMemory);
    }
    if (amVCores < 0) {
        throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting."
                + " Specified virtual cores=" + amVCores);
    }

    if (!cliParser.hasOption("jar")) {
        throw new IllegalArgumentException("No jar file specified for application master");
    }

    appMasterJar = cliParser.getOptionValue("jar");

    if (!cliParser.hasOption("shell_command") && !cliParser.hasOption("shell_script")) {
        throw new IllegalArgumentException(
                "No shell command or shell script specified to be executed by application master");
    } else if (cliParser.hasOption("shell_command") && cliParser.hasOption("shell_script")) {
        throw new IllegalArgumentException(
                "Can not specify shell_command option " + "and shell_script option at the same time");
    } else if (cliParser.hasOption("shell_command")) {
        shellCommand = cliParser.getOptionValue("shell_command");
    } else {
        shellScriptPath = cliParser.getOptionValue("shell_script");
    }
    if (cliParser.hasOption("shell_args")) {
        shellArgs = cliParser.getOptionValues("shell_args");
    }
    if (cliParser.hasOption("shell_env")) {
        String envs[] = cliParser.getOptionValues("shell_env");
        for (String env : envs) {
            env = env.trim();
            int index = env.indexOf('=');
            if (index == -1) {
                shellEnv.put(env, "");
                continue;
            }
            String key = env.substring(0, index);
            String val = "";
            if (index < (env.length() - 1)) {
                val = env.substring(index + 1);
            }
            shellEnv.put(key, val);
        }
    }
    shellCmdPriority = Integer.parseInt(cliParser.getOptionValue("shell_cmd_priority", "0"));

    containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10"));
    containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1"));
    numContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));

    if (containerMemory < 0 || containerVirtualCores < 0 || numContainers < 1) {
        throw new IllegalArgumentException("Invalid no. of containers or container memory/vcores specified,"
                + " exiting." + " Specified containerMemory=" + containerMemory + ", containerVirtualCores="
                + containerVirtualCores + ", numContainer=" + numContainers);
    }

    nodeLabelExpression = cliParser.getOptionValue("node_label_expression", null);

    clientTimeout = Integer.parseInt(cliParser.getOptionValue("timeout", "600000"));

    attemptFailuresValidityInterval = Long
            .parseLong(cliParser.getOptionValue("attempt_failures_validity_interval", "-1"));

    log4jPropFile = cliParser.getOptionValue("log_properties", "");

    // Get timeline domain options
    if (cliParser.hasOption("domain")) {
        domainId = cliParser.getOptionValue("domain");
        toCreateDomain = cliParser.hasOption("create");
        if (cliParser.hasOption("view_acls")) {
            viewACLs = cliParser.getOptionValue("view_acls");
        }
        if (cliParser.hasOption("modify_acls")) {
            modifyACLs = cliParser.getOptionValue("modify_acls");
        }
    }

    receiveLogPort = NetUtils.getFreeSocketPort();
    receiveLogHost = getLocalAddress();

    LOG.info("receiveLogHost = " + receiveLogHost + ", receiveLogPort = " + receiveLogPort);

    shellCommand += " | nc " + receiveLogHost + " " + receiveLogPort;

    return true;
}

From source file:com.mellanox.r4h.MiniDFSCluster.java

License:Apache License

protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile, boolean checkDataNodeAddrConfig)
        throws IOException {
    if (setupHostsFile) {
        String hostsFile = conf.get(DFS_HOSTS, "").trim();
        if (hostsFile.length() == 0) {
            throw new IOException("Parameter dfs.hosts is not setup in conf");
        }/*from w ww.  j  a va2s.c o m*/
        // Setup datanode in the include file, if it is defined in the conf
        String address = "127.0.0.1:" + NetUtils.getFreeSocketPort();
        if (checkDataNodeAddrConfig) {
            conf.setIfUnset(DFS_DATANODE_ADDRESS_KEY, address);
        } else {
            conf.set(DFS_DATANODE_ADDRESS_KEY, address);
        }
        addToFile(hostsFile, address);
        LOG.info("Adding datanode " + address + " to hosts file " + hostsFile);
    } else {
        if (checkDataNodeAddrConfig) {
            conf.setIfUnset(DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
        } else {
            conf.set(DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
        }
    }
    if (checkDataNodeAddrConfig) {
        conf.setIfUnset(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0");
        conf.setIfUnset(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0");
    } else {
        conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0");
        conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0");
    }
}

From source file:org.apache.tajo.ha.TestHAServiceHDFSImpl.java

License:Apache License

private void setConfiguration() {
    conf = cluster.getConfiguration();/*  w w  w . ja  v a 2 s . c o m*/

    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS, masterAddress + ":" + NetUtils.getFreeSocketPort());

    conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true);

    //Client API service RPC Server
    conf.setIntVar(TajoConf.ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);

    // Internal RPC Server
    conf.setIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
}

From source file:org.apache.tajo.master.ha.TestHAServiceHDFSImpl.java

License:Apache License

private void setConfiguration() {
    conf = cluster.getConfiguration();/*from ww  w  .j  ava 2 s  .  co  m*/

    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS,
            masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS, masterAddress + ":" + NetUtils.getFreeSocketPort());
    conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true);

    //Client API service RPC Server
    conf.setIntVar(TajoConf.ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);

    // Internal RPC Server
    conf.setIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
    conf.setIntVar(TajoConf.ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
}