List of usage examples for org.apache.hadoop.net NetUtils getHostPortString
public static String getHostPortString(InetSocketAddress addr)
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
private void createNameNode(int nnIndex, Configuration conf, int numDataNodes, boolean format, StartupOption operation, String clusterId, String nameserviceId, String nnId) throws IOException { // Format and clean out DataNode directories if (format) { DFSTestUtil.formatNameNode(conf); }/* w w w . ja va2 s . co m*/ if (operation == StartupOption.UPGRADE) { operation.setClusterId(clusterId); } // Start the NameNode after saving the default file system. String originalDefaultFs = conf.get(FS_DEFAULT_NAME_KEY); String[] args = createArgs(operation); NameNode nn = NameNode.createNameNode(args, conf); if (operation == StartupOption.RECOVER) { return; } // After the NN has started, set back the bound ports into // the conf conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnId), nn.getNameNodeAddressHostPortString()); if (nn.getHttpAddress() != null) { conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpAddress())); } if (nn.getHttpsAddress() != null) { conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY, nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpsAddress())); } DFSUtil.setGenericConf(conf, nameserviceId, nnId, DFS_NAMENODE_HTTP_ADDRESS_KEY); nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId, operation, new Configuration(conf)); // Restore the default fs name if (originalDefaultFs == null) { conf.set(FS_DEFAULT_NAME_KEY, ""); } else { conf.set(FS_DEFAULT_NAME_KEY, originalDefaultFs); } }
From source file:com.newland.bi.bigdata.hdfs.Configuration.java
License:Apache License
/** * Set the socket address for the <code>name</code> property as * a <code>host:port</code>. *///from w w w . j a va2s . c o m public void setSocketAddr(String name, InetSocketAddress addr) { set(name, NetUtils.getHostPortString(addr)); }
From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * * @param clustername/*from ww w . ja v a2 s. c o m*/ * @param clusterDirectory * @param instanceDefinition * @param debugAM * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; HoyaUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername); Configuration config = getConfig(); boolean clusterSecure = HoyaUtils.isClusterSecure(config); //create the Hoya AM provider -this helps set up the AM HoyaAMClientProvider hoyaAM = new HoyaAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; // now build up the image path // TODO: consider supporting apps that don't have an image path Path imagePath = HoyaUtils.extractImagePath(hoyaFileSystem, internalOptions); if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations hoyaAMResourceComponent = resourceOperations.getOrAddComponent(HoyaKeys.COMPONENT_AM); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, HoyaKeys.APP_TYPE, config, hoyaFileSystem, yarnClient, clusterSecure, hoyaAMResourceComponent); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT, DEFAULT_AM_RESTART_LIMIT)); hoyaFileSystem.purgeHoyaAppInstanceTempFiles(clustername); Path tempPath = hoyaFileSystem.createHoyaAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); hoyaFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", hoyaFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // conf directory setup Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(HoyaKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(HOYA_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = HoyaUtils.createLocalPath(confDir); log.debug("Copying AM configuration data from {}", localConfDirPath); remoteConfPath = new Path(clusterDirectory, HoyaKeys.SUBMITTED_CONF_DIR); HoyaUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself if (!getUsingMiniMRCluster()) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = HoyaKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = hoyaFileSystem.submitDirectory(remoteConfPath, relativeConfDir); HoyaUtils.mergeMaps(localResources, submittedConfDir); } log.debug("Copying JARs from local filesystem"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path HoyaUtils.putJar(localResources, hoyaFileSystem, this.getClass(), tempPath, libdir, SLIDER_JAR); } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); HoyaUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // add AM and provider specific artifacts to the resource map Map<String, LocalResource> providerResources; // standard AM resources hoyaAM.prepareAMAndConfigForLaunch(hoyaFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath); //add provider-specific resources provider.prepareAMAndConfigForLaunch(hoyaFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); hoyaAM.preflightValidateClusterConfiguration(hoyaFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(hoyaFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // now add the image if it was set if (hoyaFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(HoyaUtils.buildEnvMap(hoyaAMResourceComponent)); String classpath = HoyaUtils.buildClasspath(relativeConfDir, libdir, getConfig(), getUsingMiniMRCluster()); amLauncher.setEnv("CLASSPATH", classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", HoyaUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", hoyaFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress = null; try { rmSchedulerAddress = HoyaUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); CommandLineBuilder commandLine = new CommandLineBuilder(); commandLine.addJavaBinary(); // insert any JVM options); hoyaAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // add the hoya AM sevice entry point commandLine.add(HoyaAMArgs.CLASSNAME); // create action and the cluster name commandLine.add(HoyaActions.ACTION_CREATE); commandLine.add(clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_HOYA_CLUSTER_URI); commandLine.add(clusterDirectory.toUri().toString()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR); commandLine.add(rmAddr); } if (serviceArgs.getFilesystemURL() != null) { commandLine.add(Arguments.ARG_FILESYSTEM); commandLine.add(serviceArgs.getFilesystemURL().toString()); } if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over propagateConfOption(commandLine, config, HoyaXmlConfKeys.KEY_SECURITY_ENABLED); propagateConfOption(commandLine, config, DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.info("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Hoya AM gets to configure the AM requirements, not the custom provider hoyaAM.prepareAMResourceRequirements(hoyaAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_HOYA_YARN_QUEUE); amLauncher.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }
From source file:org.apache.htrace.viewer.HBaseSpanViewerServer.java
License:Apache License
void start() throws IOException { httpAddress = NetUtils// w w w .j a va 2 s. c om .createSocketAddr(conf.get(HTRACE_VIEWER_HTTP_ADDRESS_KEY, HTRACE_VIEWER_HTTP_ADDRESS_DEFAULT)); conf.set(HTRACE_VIEWER_HTTP_ADDRESS_KEY, NetUtils.getHostPortString(httpAddress)); HttpServer2.Builder builder = new HttpServer2.Builder(); builder.setName(NAME).setConf(conf); if (httpAddress.getPort() == 0) { builder.setFindPort(true); } URI uri = URI.create("http://" + NetUtils.getHostPortString(httpAddress)); builder.addEndpoint(uri); LOG.info("Starting Web-server for " + NAME + " at: " + uri); httpServer = builder.build(); httpServer.setAttribute(HTRACE_CONF_ATTR, conf); httpServer.addServlet("gettraces", HBaseSpanViewerTracesServlet.PREFIX, HBaseSpanViewerTracesServlet.class); httpServer.addServlet("getspans", HBaseSpanViewerSpansServlet.PREFIX + "/*", HBaseSpanViewerSpansServlet.class); // for webapps/htrace bundled in jar. String rb = httpServer.getClass().getClassLoader().getResource("webapps/" + NAME).toString(); httpServer.getWebAppContext().setResourceBase(rb); httpServer.start(); httpAddress = httpServer.getConnectorAddress(0); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * * @param clustername name of the cluster * @param clusterDirectory cluster dir/*w w w.java 2s . c o m*/ * @param instanceDefinition the instance definition * @param debugAM enable debug AM options * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; SliderUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); //create the Slider AM provider -this helps set up the AM SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, appOperations.getGlobalOptions().get(AgentKeys.APP_DEF)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions, applicationTags); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); sliderFileSystem.purgeAppInstanceTempFiles(clustername); Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // look for the configuration directory named on the command line boolean hasServerLog4jProperties = false; Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself boolean usingMiniMRCluster = getUsingMiniMRCluster(); if (!usingMiniMRCluster) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); SliderUtils.mergeMaps(localResources, submittedConfDir); } } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); // validate security data /* // turned off until tested SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, instanceDefinition, clustername); */ Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // standard AM resources sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); //add provider-specific resources provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // TODO: consider supporting apps that don't have an image path Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent)); ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(), usingMiniMRCluster); amLauncher.setClasspath(classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress; try { rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); // insert any JVM options); sliderAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // if the conf dir has a log4j-server.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); } // add the AM sevice entry point commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); // create action and the cluster name commandLine.add(ACTION_CREATE, clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); } if (serviceArgs.getFilesystemBinding() != null) { commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); } /** * pass the registry binding */ addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH); addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over /* addConfOptionToCLI(commandLine, config, KEY_SECURITY); */ addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.debug("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Slider AM gets to configure the AM requirements, not the custom provider sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE); if (!SliderUtils.isUnset(suppliedQueue)) { amQueue = suppliedQueue; log.info("Using queue {} for the application instance.", amQueue); } if (amQueue != null) { amLauncher.setQueue(amQueue); } // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }