List of usage examples for java.net InetSocketAddress getAddress
public final InetAddress getAddress()
From source file:org.apache.flink.runtime.taskmanager.TaskManager.java
public TaskManager(ExecutionMode executionMode, JobManagerProtocol jobManager, InputSplitProviderProtocol splitProvider, ChannelLookupProtocol channelLookup, AccumulatorProtocol accumulators, InetSocketAddress jobManagerAddress, InetAddress taskManagerBindAddress) throws Exception { if (executionMode == null || jobManager == null || splitProvider == null || channelLookup == null || accumulators == null) {/*from w w w .jav a 2s.c om*/ throw new NullPointerException(); } LOG.info("TaskManager execution mode: " + executionMode); this.executionMode = executionMode; this.jobManager = jobManager; this.lookupService = channelLookup; this.globalInputSplitProvider = splitProvider; this.accumulatorProtocolProxy = accumulators; // initialize the number of slots { int slots = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, -1); if (slots == -1) { slots = 1; LOG.info("Number of task slots not configured. Creating one task slot."); } else if (slots <= 0) { throw new Exception("Illegal value for the number of task slots: " + slots); } else { LOG.info("Creating " + slots + " task slot(s)."); } this.numberOfSlots = slots; } int ipcPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1); int dataPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, -1); if (ipcPort == -1) { ipcPort = getAvailablePort(); } if (dataPort == -1) { dataPort = getAvailablePort(); } this.localInstanceConnectionInfo = new InstanceConnectionInfo(taskManagerBindAddress, ipcPort, dataPort); LOG.info("TaskManager connection information:" + this.localInstanceConnectionInfo); // Start local RPC server, give it the number of threads as we have slots try { // some magic number for the handler threads final int numHandlers = Math.min(numberOfSlots, 2 * Hardware.getNumberCPUCores()); this.taskManagerServer = RPC.getServer(this, taskManagerBindAddress.getHostAddress(), ipcPort, numHandlers); this.taskManagerServer.start(); } catch (IOException e) { LOG.error("Failed to start TaskManager server. " + e.getMessage(), e); throw new Exception("Failed to start taskmanager server. " + e.getMessage(), e); } // Load profiler if it should be used if (GlobalConfiguration.getBoolean(ProfilingUtils.ENABLE_PROFILING_KEY, false)) { final String profilerClassName = GlobalConfiguration.getString(ProfilingUtils.TASKMANAGER_CLASSNAME_KEY, "org.apache.flink.runtime.profiling.impl.TaskManagerProfilerImpl"); this.profiler = ProfilingUtils.loadTaskManagerProfiler(profilerClassName, jobManagerAddress.getAddress(), this.localInstanceConnectionInfo); if (this.profiler == null) { LOG.error("Cannot find class name for the profiler."); } else { LOG.info("Profiling of jobs is enabled."); } } else { this.profiler = null; LOG.info("Profiling of jobs is disabled."); } // Get the directory for storing temporary files final String[] tmpDirPaths = GlobalConfiguration .getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH) .split(",|" + File.pathSeparator); checkTempDirs(tmpDirPaths); int numBuffers = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS); int bufferSize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE); // Initialize the channel manager try { NetworkConnectionManager networkConnectionManager = null; switch (executionMode) { case LOCAL: networkConnectionManager = new LocalConnectionManager(); break; case CLUSTER: int numInThreads = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_NET_NUM_IN_THREADS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_IN_THREADS); int numOutThreads = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_NET_NUM_OUT_THREADS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS); int lowWaterMark = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_NET_NETTY_LOW_WATER_MARK, ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK); int highWaterMark = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK, ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK); networkConnectionManager = new NettyConnectionManager(localInstanceConnectionInfo.address(), localInstanceConnectionInfo.dataPort(), bufferSize, numInThreads, numOutThreads, lowWaterMark, highWaterMark); break; } channelManager = new ChannelManager(lookupService, localInstanceConnectionInfo, numBuffers, bufferSize, networkConnectionManager); } catch (IOException ioe) { LOG.error(StringUtils.stringifyException(ioe)); throw new Exception("Failed to instantiate ChannelManager.", ioe); } // initialize the memory manager { // Check whether the memory size has been explicitly configured. final long configuredMemorySize = GlobalConfiguration .getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1); final long memorySize; if (configuredMemorySize == -1) { // no manually configured memory. take a relative fraction of the free heap space float fraction = GlobalConfiguration.getFloat(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY, ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION); memorySize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * fraction); LOG.info("Using " + fraction + " of the free heap space for managed memory."); } else if (configuredMemorySize <= 0) { throw new Exception("Invalid value for Memory Manager memory size: " + configuredMemorySize); } else { memorySize = configuredMemorySize << 20; } final int pageSize = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE); // Initialize the memory manager LOG.info("Initializing memory manager with " + (memorySize >>> 20) + " megabytes of memory. " + "Page size is " + pageSize + " bytes."); try { @SuppressWarnings("unused") final boolean lazyAllocation = GlobalConfiguration.getBoolean( ConfigConstants.TASK_MANAGER_MEMORY_LAZY_ALLOCATION_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_LAZY_ALLOCATION); this.memoryManager = new DefaultMemoryManager(memorySize, this.numberOfSlots, pageSize); } catch (Throwable t) { LOG.error( "Unable to initialize memory manager with " + (memorySize >>> 20) + " megabytes of memory.", t); throw new Exception("Unable to initialize memory manager.", t); } } this.hardwareDescription = HardwareDescription.extractFromSystem(this.memoryManager.getMemorySize()); // Determine the port of the BLOB server and register it with the library cache manager { final int blobPort = this.jobManager.getBlobServerPort(); if (blobPort == -1) { LOG.warn("Unable to determine BLOB server address: User library download will not be available"); this.libraryCacheManager = new FallbackLibraryCacheManager(); } else { final InetSocketAddress blobServerAddress = new InetSocketAddress(jobManagerAddress.getAddress(), blobPort); LOG.info("Determined BLOB server address to be " + blobServerAddress); this.libraryCacheManager = new BlobLibraryCacheManager(new BlobCache(blobServerAddress), GlobalConfiguration.getConfiguration()); } } this.ioManager = new IOManagerAsync(tmpDirPaths); // start the heart beats { final long interval = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_HEARTBEAT_INTERVAL_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_HEARTBEAT_INTERVAL); this.heartbeatThread = new Thread() { @Override public void run() { registerAndRunHeartbeatLoop(interval, MAX_LOST_HEART_BEATS); } }; this.heartbeatThread.setName("Heartbeat Thread"); this.heartbeatThread.start(); } // -------------------------------------------------------------------- // Memory Usage // -------------------------------------------------------------------- final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); final List<GarbageCollectorMXBean> gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans(); LOG.info(getMemoryUsageStatsAsString(memoryMXBean)); boolean startMemoryUsageLogThread = GlobalConfiguration.getBoolean( ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD, ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD); if (startMemoryUsageLogThread) { final int logIntervalMs = GlobalConfiguration.getInteger( ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS, ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS); new Thread(new Runnable() { @Override public void run() { try { while (!isShutDown()) { Thread.sleep(logIntervalMs); LOG.info(getMemoryUsageStatsAsString(memoryMXBean)); LOG.info(getGarbageCollectorStatsAsString(gcMXBeans)); } } catch (InterruptedException e) { LOG.warn("Unexpected interruption of memory usage logger thread."); } } }).start(); } }
From source file:com.bytelightning.opensource.pokerface.PokerFace.java
/** * Configures all the needed components, but does not actually start the server. * @param config Contains all information needed to fully wire up the http, https, and httpclient components of this reverse proxy. * @throws Exception Yeah, a lot can go wrong here, but at least it will be caught immediately :-) *///ww w .j a v a 2 s .c om public void config(HierarchicalConfiguration config) throws Exception { List<HierarchicalConfiguration> lconf; HttpAsyncRequester executor = null; BasicNIOConnPool connPool = null; ObjectPool<ByteBuffer> byteBufferPool = null; LinkedHashMap<String, TargetDescriptor> mappings = null; ConcurrentMap<String, HttpHost> hosts = null; handlerRegistry = new UriHttpAsyncRequestHandlerMapper(); // Initialize the keystore (if one was specified) KeyStore keystore = null; char[] keypass = null; String keystoreUri = config.getString("keystore"); if ((keystoreUri != null) && (keystoreUri.trim().length() > 0)) { Path keystorePath = Utils.MakePath(keystoreUri); if (!Files.exists(keystorePath)) throw new ConfigurationException("Keystore does not exist."); if (Files.isDirectory(keystorePath)) throw new ConfigurationException("Keystore is not a file"); String storepass = config.getString("storepass"); if ((storepass != null) && "null".equals(storepass)) storepass = null; keystore = KeyStore.getInstance(KeyStore.getDefaultType()); try (InputStream keyStoreStream = Files.newInputStream(keystorePath)) { keystore.load(keyStoreStream, storepass == null ? null : storepass.trim().toCharArray()); } catch (IOException ex) { Logger.error("Unable to load https server keystore from " + keystoreUri); return; } keypass = config.getString("keypass").trim().toCharArray(); } // Wire up the listening reactor lconf = config.configurationsAt("server"); if ((lconf == null) || (lconf.size() != 1)) throw new ConfigurationException("One (and only one) server configuration element is allowed."); else { Builder builder = IOReactorConfig.custom(); builder.setIoThreadCount(ComputeReactorProcessors(config.getDouble("server[@cpu]", 0.667))); builder.setSoTimeout(config.getInt("server[@soTimeout]", 0)); builder.setSoLinger(config.getInt("server[@soLinger]", -1)); builder.setSoReuseAddress(true); builder.setTcpNoDelay(false); builder.setSelectInterval(100); IOReactorConfig rconfig = builder.build(); Logger.info("Configuring server with options: " + rconfig.toString()); listeningReactor = new DefaultListeningIOReactor(rconfig); lconf = config.configurationsAt("server.listen"); InetSocketAddress addr; boolean hasNonWildcardSecure = false; LinkedHashMap<SocketAddress, SSLContext> addrSSLContext = new LinkedHashMap<SocketAddress, SSLContext>(); if ((lconf == null) || (lconf.size() == 0)) { addr = new InetSocketAddress("127.0.0.1", 8080); ListenerEndpoint ep = listeningReactor.listen(addr); Logger.warn("Configured " + ep.getAddress()); } else { TrustManager[] trustManagers = null; KeyManagerFactory kmf = null; // Create all the specified listeners. for (HierarchicalConfiguration hc : lconf) { String addrStr = hc.getString("[@address]"); if ((addrStr == null) || (addrStr.length() == 0)) addrStr = "0.0.0.0"; String alias = hc.getString("[@alias]"); int port = hc.getInt("[@port]", alias != null ? 443 : 80); addr = new InetSocketAddress(addrStr, port); ListenerEndpoint ep = listeningReactor.listen(addr); String protocol = hc.containsKey("[@protocol]") ? hc.getString("[@protocol]") : null; Boolean secure = hc.containsKey("[@secure]") ? hc.getBoolean("[@secure]") : null; if ((alias != null) && (secure == null)) secure = true; if ((protocol != null) && (secure == null)) secure = true; if ((secure != null) && secure) { if (protocol == null) protocol = "TLS"; if (keystore == null) throw new ConfigurationException( "An https listening socket was requested, but no keystore was specified."); if (kmf == null) { kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(keystore, keypass); } // Are we going to trust all clients or just specific ones? if (hc.getBoolean("[@trustAny]", true)) trustManagers = new TrustManager[] { new X509TrustAllManager() }; else { TrustManagerFactory instance = TrustManagerFactory .getInstance(TrustManagerFactory.getDefaultAlgorithm()); instance.init(keystore); trustManagers = instance.getTrustManagers(); } KeyManager[] keyManagers = kmf.getKeyManagers(); if (alias != null) for (int i = 0; i < keyManagers.length; i++) { if (keyManagers[i] instanceof X509ExtendedKeyManager) keyManagers[i] = new PokerFaceKeyManager(alias, (X509ExtendedKeyManager) keyManagers[i]); } SSLContext sslCtx = SSLContext.getInstance(protocol); sslCtx.init(keyManagers, trustManagers, new SecureRandom()); if (addr.getAddress().isAnyLocalAddress()) { // This little optimization helps us respond faster for every connection as we don't have to extrapolate a local connection address to wild card. for (Enumeration<NetworkInterface> en = NetworkInterface.getNetworkInterfaces(); en .hasMoreElements();) { NetworkInterface intf = en.nextElement(); for (Enumeration<InetAddress> enumIpAddr = intf.getInetAddresses(); enumIpAddr .hasMoreElements();) { addr = new InetSocketAddress(enumIpAddr.nextElement(), port); addrSSLContext.put(addr, sslCtx); } } } else { addrSSLContext.put(addr, sslCtx); hasNonWildcardSecure = true; } } Logger.warn("Configured " + (alias == null ? "" : (protocol + " on")) + ep.getAddress()); } } // We will need an HTTP protocol processor for the incoming connections String serverAgent = config.getString("server.serverAgent", "PokerFace/" + Utils.Version); HttpProcessor inhttpproc = new ImmutableHttpProcessor( new HttpResponseInterceptor[] { new ResponseDateInterceptor(), new ResponseServer(serverAgent), new ResponseContent(), new ResponseConnControl() }); HttpAsyncService serviceHandler = new HttpAsyncService(inhttpproc, new DefaultConnectionReuseStrategy(), null, handlerRegistry, null) { public void exception(final NHttpServerConnection conn, final Exception cause) { Logger.warn(cause.getMessage()); super.exception(conn, cause); } }; if (addrSSLContext.size() > 0) { final SSLContext defaultCtx = addrSSLContext.values().iterator().next(); final Map<SocketAddress, SSLContext> sslMap; if ((!hasNonWildcardSecure) || (addrSSLContext.size() == 1)) sslMap = null; else sslMap = addrSSLContext; listeningDispatcher = new DefaultHttpServerIODispatch(serviceHandler, new SSLNHttpServerConnectionFactory(defaultCtx, null, ConnectionConfig.DEFAULT) { protected SSLIOSession createSSLIOSession(IOSession iosession, SSLContext sslcontext, SSLSetupHandler sslHandler) { SSLIOSession retVal; SSLContext sktCtx = sslcontext; if (sslMap != null) { SocketAddress la = iosession.getLocalAddress(); if (la != null) { sktCtx = sslMap.get(la); if (sktCtx == null) sktCtx = sslcontext; } retVal = new SSLIOSession(iosession, SSLMode.SERVER, sktCtx, sslHandler); } else retVal = super.createSSLIOSession(iosession, sktCtx, sslHandler); if (sktCtx != null) retVal.setAttribute("com.bytelightning.opensource.pokerface.secure", true); return retVal; } }); } else listeningDispatcher = new DefaultHttpServerIODispatch(serviceHandler, ConnectionConfig.DEFAULT); } // Configure the httpclient reactor that will be used to do reverse proxing to the specified targets. lconf = config.configurationsAt("targets"); if ((lconf != null) && (lconf.size() > 0)) { HierarchicalConfiguration conf = lconf.get(0); Builder builder = IOReactorConfig.custom(); builder.setIoThreadCount(ComputeReactorProcessors(config.getDouble("targets[@cpu]", 0.667))); builder.setSoTimeout(conf.getInt("targets[@soTimeout]", 0)); builder.setSoLinger(config.getInt("targets[@soLinger]", -1)); builder.setConnectTimeout(conf.getInt("targets[@connectTimeout]", 0)); builder.setSoReuseAddress(true); builder.setTcpNoDelay(false); connectingReactor = new DefaultConnectingIOReactor(builder.build()); final int bufferSize = conf.getInt("targets[@bufferSize]", 1024) * 1024; byteBufferPool = new SoftReferenceObjectPool<ByteBuffer>(new BasePooledObjectFactory<ByteBuffer>() { @Override public ByteBuffer create() throws Exception { return ByteBuffer.allocateDirect(bufferSize); } @Override public PooledObject<ByteBuffer> wrap(ByteBuffer buffer) { return new DefaultPooledObject<ByteBuffer>(buffer); } }); KeyManager[] keyManagers = null; TrustManager[] trustManagers = null; if (keystore != null) { KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(keystore, keypass); keyManagers = kmf.getKeyManagers(); } // Will the httpclient's trust any remote target, or only specific ones. if (conf.getBoolean("targets[@trustAny]", false)) trustManagers = new TrustManager[] { new X509TrustAllManager() }; else if (keystore != null) { TrustManagerFactory instance = TrustManagerFactory .getInstance(TrustManagerFactory.getDefaultAlgorithm()); instance.init(keystore); trustManagers = instance.getTrustManagers(); } SSLContext clientSSLContext = SSLContext.getInstance(conf.getString("targets[@protocol]", "TLS")); clientSSLContext.init(keyManagers, trustManagers, new SecureRandom()); // Setup an SSL capable connection pool for the httpclients. connPool = new BasicNIOConnPool(connectingReactor, new BasicNIOConnFactory(clientSSLContext, null, ConnectionConfig.DEFAULT), conf.getInt("targets[@connectTimeout]", 0)); connPool.setMaxTotal(conf.getInt("targets[@connMaxTotal]", 1023)); connPool.setDefaultMaxPerRoute(conf.getInt("targets[@connMaxPerRoute]", 1023)); // Set up HTTP protocol processor for outgoing connections String userAgent = conf.getString("targets.userAgent", "PokerFace/" + Utils.Version); HttpProcessor outhttpproc = new ImmutableHttpProcessor(new HttpRequestInterceptor[] { new RequestContent(), new RequestTargetHost(), new RequestConnControl(), new RequestUserAgent(userAgent), new RequestExpectContinue(true) }); executor = new HttpAsyncRequester(outhttpproc, new DefaultConnectionReuseStrategy()); // Now set up all the configured targets. mappings = new LinkedHashMap<String, TargetDescriptor>(); hosts = new ConcurrentHashMap<String, HttpHost>(); String[] scheme = { null }; String[] host = { null }; int[] port = { 0 }; String[] path = { null }; int[] stripPrefixCount = { 0 }; for (HierarchicalConfiguration targetConfig : conf.configurationsAt("target")) { String match = targetConfig.getString("[@pattern]"); if ((match == null) || (match.trim().length() < 1)) { Logger.error("Unable to configure target; Invalid url match pattern"); continue; } String key = RequestForTargetConsumer.UriToTargetKey(targetConfig.getString("[@url]"), scheme, host, port, path, stripPrefixCount); if (key == null) { Logger.error("Unable to configure target"); continue; } HttpHost targetHost = hosts.get(key); if (targetHost == null) { targetHost = new HttpHost(host[0], port[0], scheme[0]); hosts.put(key, targetHost); } TargetDescriptor desc = new TargetDescriptor(targetHost, path[0], stripPrefixCount[0]); mappings.put(match, desc); } connectionDispatcher = new DefaultHttpClientIODispatch(new HttpAsyncRequestExecutor(), ConnectionConfig.DEFAULT); } // Allocate the script map which will be populated by it's own executor thread. if (config.containsKey("scripts.rootDirectory")) { Path tmp = Utils.MakePath(config.getProperty("scripts.rootDirectory")); if (!Files.exists(tmp)) throw new FileNotFoundException("Scripts directory does not exist."); if (!Files.isDirectory(tmp)) throw new FileNotFoundException("'scripts' path is not a directory."); scripts = new ConcurrentSkipListMap<String, ScriptObjectMirror>(); boolean watch = config.getBoolean("scripts.dynamicWatch", false); List<Path> jsLibs; Object prop = config.getProperty("scripts.library"); if (prop != null) { jsLibs = new ArrayList<Path>(); if (prop instanceof Collection<?>) { @SuppressWarnings("unchecked") Collection<Object> oprop = (Collection<Object>) prop; for (Object obj : oprop) jsLibs.add(Utils.MakePath(obj)); } else { jsLibs.add(Utils.MakePath(prop)); } } else jsLibs = null; lconf = config.configurationsAt("scripts.scriptConfig"); if (lconf != null) { if (lconf.size() > 1) throw new ConfigurationException("Only one scriptConfig element is allowed."); if (lconf.size() == 0) lconf = null; } HierarchicalConfiguration scriptConfig; if (lconf == null) scriptConfig = new HierarchicalConfiguration(); else scriptConfig = lconf.get(0); scriptConfig.setProperty("pokerface.scripts.rootDirectory", tmp.toString()); configureScripts(jsLibs, scriptConfig, tmp, watch); if (watch) ScriptDirectoryWatcher = new DirectoryWatchService(); } // Configure the static file directory (if any) Path staticFilesPath = null; if (config.containsKey("files.rootDirectory")) { Path tmp = Utils.MakePath(config.getProperty("files.rootDirectory")); if (!Files.exists(tmp)) throw new FileNotFoundException("Files directory does not exist."); if (!Files.isDirectory(tmp)) throw new FileNotFoundException("'files' path is not a directory."); staticFilesPath = tmp; List<HierarchicalConfiguration> mimeEntries = config.configurationsAt("files.mime-entry"); if (mimeEntries != null) { for (HierarchicalConfiguration entry : mimeEntries) { entry.setDelimiterParsingDisabled(true); String type = entry.getString("[@type]", "").trim(); if (type.length() == 0) throw new ConfigurationException("Invalid mime type entry"); String extensions = entry.getString("[@extensions]", "").trim(); if (extensions.length() == 0) throw new ConfigurationException("Invalid mime extensions for: " + type); ScriptHelperImpl.AddMimeEntry(type, extensions); } } } handlerRegistry.register("/*", new RequestHandler(executor, connPool, byteBufferPool, staticFilesPath, mappings, scripts != null ? Collections.unmodifiableNavigableMap(scripts) : null, config.getBoolean("scripts.allowScriptsToSpecifyDynamicHosts", false) ? hosts : null)); }
From source file:net.spfbl.http.ServerHTTP.java
private static Client getClient(HttpExchange exchange) { InetSocketAddress socketAddress = exchange.getRemoteAddress(); InetAddress address = socketAddress.getAddress(); return Client.get(address); }
From source file:net.spfbl.http.ServerHTTP.java
private static String getRemoteAddress(HttpExchange exchange) { InetSocketAddress socketAddress = exchange.getRemoteAddress(); InetAddress address = socketAddress.getAddress(); return address.getHostAddress(); }
From source file:eu.stratosphere.nephele.taskmanager.TaskManager.java
/** * Constructs a new task manager, starts its IPC service and attempts to discover the job manager to * receive an initial configuration. All parameters are obtained from the * {@link GlobalConfiguration}, which must be loaded prior to instantiating the task manager. *//*from w w w.java2 s .c o m*/ public TaskManager() throws Exception { LOG.info("TaskManager started as user " + UserGroupInformation.getCurrentUser().getShortUserName()); LOG.info("User system property: " + System.getProperty("user.name")); // IMPORTANT! At this point, the GlobalConfiguration must have been read! final InetSocketAddress jobManagerAddress; { LOG.info("Reading location of job manager from configuration"); final String address = GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null); final int port = GlobalConfiguration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT); if (address == null) { throw new Exception("Job manager address not configured in the GlobalConfiguration."); } // Try to convert configured address to {@link InetAddress} try { final InetAddress tmpAddress = InetAddress.getByName(address); jobManagerAddress = new InetSocketAddress(tmpAddress, port); } catch (UnknownHostException e) { LOG.fatal("Could not resolve JobManager host name."); throw new Exception("Could not resolve JobManager host name: " + e.getMessage(), e); } LOG.info("Connecting to JobManager at: " + jobManagerAddress); } // Create RPC connection to the JobManager try { this.jobManager = RPC.getProxy(JobManagerProtocol.class, jobManagerAddress, NetUtils.getSocketFactory()); } catch (IOException e) { LOG.fatal("Could not connect to the JobManager: " + e.getMessage(), e); throw new Exception("Failed to initialize connection to JobManager: " + e.getMessage(), e); } int ipcPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1); int dataPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, -1); if (ipcPort == -1) { ipcPort = getAvailablePort(); } if (dataPort == -1) { dataPort = getAvailablePort(); } // Determine our own public facing address and start the server { final InetAddress taskManagerAddress; try { taskManagerAddress = getTaskManagerAddress(jobManagerAddress); } catch (Exception e) { throw new RuntimeException("The TaskManager failed to determine its own network address.", e); } this.localInstanceConnectionInfo = new InstanceConnectionInfo(taskManagerAddress, ipcPort, dataPort); LOG.info("TaskManager connection information:" + this.localInstanceConnectionInfo); // Start local RPC server try { this.taskManagerServer = RPC.getServer(this, taskManagerAddress.getHostAddress(), ipcPort, IPC_HANDLER_COUNT); this.taskManagerServer.start(); } catch (IOException e) { LOG.fatal("Failed to start TaskManager server. " + e.getMessage(), e); throw new Exception("Failed to start taskmanager server. " + e.getMessage(), e); } } // Try to create local stub of the global input split provider try { this.globalInputSplitProvider = RPC.getProxy(InputSplitProviderProtocol.class, jobManagerAddress, NetUtils.getSocketFactory()); } catch (IOException e) { LOG.fatal(e.getMessage(), e); throw new Exception("Failed to initialize connection to global input split provider: " + e.getMessage(), e); } // Try to create local stub for the lookup service try { this.lookupService = RPC.getProxy(ChannelLookupProtocol.class, jobManagerAddress, NetUtils.getSocketFactory()); } catch (IOException e) { LOG.fatal(e.getMessage(), e); throw new Exception("Failed to initialize channel lookup protocol. " + e.getMessage(), e); } // Try to create local stub for the accumulators try { this.accumulatorProtocolProxy = RPC.getProxy(AccumulatorProtocol.class, jobManagerAddress, NetUtils.getSocketFactory()); } catch (IOException e) { LOG.fatal("Failed to initialize accumulator protocol: " + e.getMessage(), e); throw new Exception("Failed to initialize accumulator protocol: " + e.getMessage(), e); } // Load profiler if it should be used if (GlobalConfiguration.getBoolean(ProfilingUtils.ENABLE_PROFILING_KEY, false)) { final String profilerClassName = GlobalConfiguration.getString(ProfilingUtils.TASKMANAGER_CLASSNAME_KEY, "eu.stratosphere.nephele.profiling.impl.TaskManagerProfilerImpl"); this.profiler = ProfilingUtils.loadTaskManagerProfiler(profilerClassName, jobManagerAddress.getAddress(), this.localInstanceConnectionInfo); if (this.profiler == null) { LOG.error("Cannot find class name for the profiler."); } else { LOG.info("Profiling of jobs is enabled."); } } else { this.profiler = null; LOG.info("Profiling of jobs is disabled."); } // Get the directory for storing temporary files final String[] tmpDirPaths = GlobalConfiguration .getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH) .split(",|" + File.pathSeparator); checkTempDirs(tmpDirPaths); final int pageSize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE); // Initialize network buffer pool int numBuffers = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS); int bufferSize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE); int numInThreads = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETTY_NUM_IN_THREADS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETTY_NUM_IN_THREADS); int numOutThreads = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETTY_NUM_OUT_THREADS_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_NETTY_NUM_OUT_THREADS); int lowWaterMark = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETTY_LOW_WATER_MARK, ConfigConstants.DEFAULT_TASK_MANAGER_NETTY_LOW_WATER_MARK); int highWaterMark = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETTY_HIGH_WATER_MARK, ConfigConstants.DEFAULT_TASK_MANAGER_NETTY_HIGH_WATER_MARK); // Initialize the channel manager try { this.channelManager = new ChannelManager(this.lookupService, this.localInstanceConnectionInfo, numBuffers, bufferSize, numInThreads, numOutThreads, lowWaterMark, highWaterMark); } catch (IOException ioe) { LOG.error(StringUtils.stringifyException(ioe)); throw new Exception("Failed to instantiate Byte-buffered channel manager. " + ioe.getMessage(), ioe); } { HardwareDescription resources = HardwareDescriptionFactory.extractFromSystem(); // Check whether the memory size has been explicitly configured. if so that overrides the default mechanism // of taking as much as is mentioned in the hardware description long memorySize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1); if (memorySize > 0) { // manually configured memory size. override the value in the hardware config resources = HardwareDescriptionFactory.construct(resources.getNumberOfCPUCores(), resources.getSizeOfPhysicalMemory(), memorySize * 1024L * 1024L); } this.hardwareDescription = resources; // Initialize the memory manager LOG.info("Initializing memory manager with " + (resources.getSizeOfFreeMemory() >>> 20) + " megabytes of memory. " + "Page size is " + pageSize + " bytes."); try { @SuppressWarnings("unused") final boolean lazyAllocation = GlobalConfiguration.getBoolean( ConfigConstants.TASK_MANAGER_MEMORY_LAZY_ALLOCATION_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_LAZY_ALLOCATION); this.memoryManager = new DefaultMemoryManager(resources.getSizeOfFreeMemory(), pageSize); } catch (Throwable t) { LOG.fatal("Unable to initialize memory manager with " + (resources.getSizeOfFreeMemory() >>> 20) + " megabytes of memory.", t); throw new Exception("Unable to initialize memory manager.", t); } } this.ioManager = new IOManager(tmpDirPaths); this.heartbeatThread = new Thread() { @Override public void run() { runHeartbeatLoop(); } }; this.heartbeatThread.setName("Heartbeat Thread"); this.heartbeatThread.start(); }