List of usage examples for org.apache.hadoop.net NetUtils getSocketFactory
public static SocketFactory getSocketFactory(Configuration conf, Class<?> clazz)
From source file:com.chinamobile.bcbsp.client.BSPJobClient.java
License:Apache License
/** * Ensure fresh jobSubmitClient./* w ww .j a va2 s . c o m*/ */ public void ensureFreshjJobSubmitClient() { try { if (bspzk != null) { Stat s = null; int count = 0; int max = 3; while (s == null && count < max) { count++; Thread.sleep(500); s = bspzk.exists(Constants.BSPCONTROLLER_LEADER, true); } if (s != null) { String controllerAddr = getData(Constants.BSPCONTROLLER_LEADER); InetSocketAddress newAddr = NetUtils.createSocketAddr(controllerAddr); if (this.bspControllerAddr == null || !this.bspControllerAddr.equals(newAddr)) { this.bspControllerAddr = newAddr; // establish connection to new BspController again conf.set("ipc.client.connect.max.retries", "0"); this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy(JobSubmissionProtocol.class, BSPRPCProtocolVersion.versionID, bspControllerAddr, conf, NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); LOG.info("Now connected to " + this.bspControllerAddr.toString()); } } } } catch (Exception e) { // LOG.warn("lost connection to " + this.bspControllerAddr.toString()); // LOG.error("[ensureFreshjJobSubmitClient]", e); throw new RuntimeException("lost connection to bspControllerAddr ", e); } }
From source file:com.intel.hadoopRPCBenchmark.protocol.BenchmarkEngineProtocolClientSideTranslatorPB.java
License:Apache License
public BenchmarkEngineProtocolClientSideTranslatorPB(InetSocketAddress nameNodeAddr, Configuration conf) throws IOException { RPC.setProtocolEngine(conf, BenchmarkEngineProtocolPB.class, ProtobufRpcEngine.class); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); rpcProxy = RPC.getProxy(BenchmarkEngineProtocolPB.class, RPC.getProtocolVersion(BenchmarkEngineProtocolPB.class), nameNodeAddr, ugi, conf, NetUtils.getSocketFactory(conf, BenchmarkEngineProtocolPB.class)); }
From source file:com.mellanox.r4h.DFSClient.java
License:Apache License
/** * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode. * If HA is enabled and a positive value is set for {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the * configuration, the DFSClient will use {@link LossyRetryInvocationHandler} as its RetryInvocationHandler. Otherwise one of nameNodeUri or * rpcNamenode/*from ww w. j a va 2s. c om*/ * must be null. */ @VisibleForTesting public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, Configuration conf, FileSystem.Statistics stats) throws IOException { SpanReceiverHost.get(conf, DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX); traceSampler = new SamplerBuilder(TraceUtils.wrapHadoopConf(DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX, conf)) .build(); // Copy only the required DFSClient configuration this.dfsClientConf = new DFSClientConfBridge2_7(conf); if (this.dfsClientConf.isUseLegacyBlockReaderLocal()) { LOG.debug("Using legacy short-circuit local reads."); } this.conf = conf; this.stats = stats; this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); this.ugi = UserGroupInformation.getCurrentUser(); this.authority = nameNodeUri == null ? "null" : nameNodeUri.getAuthority(); this.clientName = "DFSClient_" + dfsClientConf.getTaskId() + "_" + DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId(); provider = DFSUtil.createKeyProvider(conf); if (LOG.isDebugEnabled()) { if (provider == null) { LOG.debug("No KeyProvider found."); } else { LOG.debug("Found KeyProvider: " + provider.toString()); } } int numResponseToDrop = conf.getInt(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY, DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT); NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null; AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false); if (numResponseToDrop > 0) { // This case is used for testing. LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY + " is set to " + numResponseToDrop + ", this hacked client will proactively drop responses"); proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf, nameNodeUri, ClientProtocol.class, numResponseToDrop, nnFallbackToSimpleAuth); } if (proxyInfo != null) { this.dtService = proxyInfo.getDelegationTokenService(); this.namenode = proxyInfo.getProxy(); } else if (rpcNamenode != null) { // This case is used for testing. Preconditions.checkArgument(nameNodeUri == null); this.namenode = rpcNamenode; dtService = null; } else { Preconditions.checkArgument(nameNodeUri != null, "null URI"); proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class, nnFallbackToSimpleAuth); this.dtService = proxyInfo.getDelegationTokenService(); this.namenode = proxyInfo.getProxy(); } String localInterfaces[] = conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES); localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces); if (LOG.isDebugEnabled() && 0 != localInterfaces.length) { LOG.debug("Using local interfaces [" + Joiner.on(',').join(localInterfaces) + "] with addresses [" + Joiner.on(',').join(localInterfaceAddrs) + "]"); } Boolean readDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false); Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ? null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0); Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false); this.defaultReadCachingStrategy = new CachingStrategy(readDropBehind, readahead); this.defaultWriteCachingStrategy = new CachingStrategy(writeDropBehind, readahead); this.clientContext = ClientContext.get(conf.get(DFS_CLIENT_CONTEXT, DFS_CLIENT_CONTEXT_DEFAULT), dfsClientConf); this.hedgedReadThresholdMillis = conf.getLong(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS, DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS); int numThreads = conf.getInt(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE, DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE); if (numThreads > 0) { this.initThreadsNumForHedgedReads(numThreads); } this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnFallbackToSimpleAuth); }
From source file:org.apache.ambari.servicemonitor.utils.MonitorUtils.java
License:Apache License
public static AdminOperationsProtocol createJTAdminProxy(InetSocketAddress addr, Configuration conf) throws IOException { return (AdminOperationsProtocol) RPC.getProxy(AdminOperationsProtocol.class, AdminOperationsProtocol.versionID, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getSocketFactory(conf, AdminOperationsProtocol.class)); }
From source file:org.apache.hama.bsp.BSPJobClient.java
License:Apache License
public void init(Configuration conf) throws IOException { String masterAdress = conf.get("bsp.master.address"); if (masterAdress != null && !masterAdress.equals("local")) { this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy(JobSubmissionProtocol.class, HamaRPCProtocolVersion.versionID, BSPMaster.getAddress(conf), conf, NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); } else {// w w w. jav a 2s . c o m LOG.debug("Using local BSP runner."); this.jobSubmitClient = new LocalBSPRunner(conf); } }
From source file:org.apache.ratis.hadooprpc.Proxy.java
License:Apache License
public static <PROTOCOL> PROTOCOL getProxy(Class<PROTOCOL> clazz, String addressStr, Configuration conf) throws IOException { RPC.setProtocolEngine(conf, clazz, ProtobufRpcEngineShaded.class); return RPC.getProxy(clazz, RPC.getProtocolVersion(clazz), org.apache.ratis.util.NetUtils.createSocketAddr(addressStr), UserGroupInformation.getCurrentUser(), conf, NetUtils.getSocketFactory(conf, clazz)); }
From source file:org.cstor.cproc.cloudComputingFramework.CProcFramework.java
License:Apache License
public static void main(String argv[]) throws Exception { LOG.info("CProcFramework version for hadoop-0.20.2-patch : 32"); //GCThreadGC//from ww w.j av a 2 s .c o m Thread th = new Thread(new GCThread()); th.start(); //LOG.info("CProcFramework [NameNode IP:Port]"); StringUtils.startupShutdownMessage(CProcFramework.class, argv, LOG); //hdfs-site.xml,core-site.xml CProcConfiguration cprocConf = CProcConfiguration.getCProcConfiguration(); RpcConfiguration rpcConf = RpcConfiguration.getRpcConfiguration(); setParamToRpcConf(cprocConf, rpcConf); CProcFramework cpf = null; ClientProtocol nameNode = null; ClientDatanodeProtocol dataNode = null; InetSocketAddress dataNodeAddr = null; InetSocketAddress nameNodeAddr = null; try { boolean isNN = true; //LOG.info("conf.get(\"fs.default.name\") == " + conf.get("cProc.name.") ); //wzt ??? ?? net0 192.168.139.21 9000 if (argv.length >= 3) { network = argv[0]; namenodeAddress = argv[1]; port = argv[2]; } else { LOG.info("CProcFramework [network adapter] [namenode Address]"); System.exit(0); } LOG.info("network = " + network); try { nameNodeAddr = NetUtils.createSocketAddr(namenodeAddress); } catch (IllegalArgumentException e) { //nameNodeAddr = NetUtils.createSocketAddr(argv[0]); } LOG.info("NameNodeAddr = " + nameNodeAddr); //-----------------------get all DN and get conf of NN---------------------------------- ArrayList<String> DNIP = new ArrayList<String>(); // Configuration confInFunc = new Configuration(); nameNode = (ClientProtocol) RPC.getProxy(ClientProtocol.class, ClientProtocol.versionID, nameNodeAddr, rpcConf, NetUtils.getSocketFactory(rpcConf, ClientProtocol.class)); DatanodeDescriptor[] dds = nameNode.getAllDatanode(); // LOG.info("=========="); for (DatanodeDescriptor dd : dds) { LOG.info(dd.getHost()); //LOG.info(dd.getHostName()); //LOG.info(dd.getDatanodeReport()); DNIP.add(dd.getHost()); } //LOG.info("=========="); //conf = nameNode.getConfiguration(); setParamToCprocConf(nameNode.getConfiguration(), rpcConf); LOG.info("getHostIP.getLocalIP() = " + getHostIPcProc.getLocalIP(network)); //-----------------------if this node is a DN get conf---------------------------------- if (DNIP.contains(getHostIPcProc.getLocalIP(network))) { dataNodeAddr = NetUtils .createSocketAddr("127.0.0.1:" + rpcConf.get("dfs.datanode.ipc.address").split(":")[1]); dataNode = (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, dataNodeAddr, rpcConf, NetUtils.getSocketFactory(rpcConf, ClientDatanodeProtocol.class)); //conf = dataNode.getConfiguration(); LOG.info("This is DataNode!!"); isNN = false; } else { LOG.info("This is NameNode!!"); isNN = true; } cpf = createCProcFramework(nameNode, dataNode, cprocConf, rpcConf, isNN, nameNodeAddr); cpf.waitForStop(); } catch (Throwable e) { LOG.error(StringUtils.stringifyException(e)); System.exit(-1); } finally { //20131210RPC // if(nameNode != null){ // RPC.stopProxy(nameNode); // } // if(dataNode != null){ // RPC.stopProxy(dataNode); // } } }
From source file:skewtune.mapreduce.STJobTracker.java
License:Apache License
@SuppressWarnings("unchecked") STJobTracker(final JobConf conf, String jobtrackerIndentifier) throws IOException, InterruptedException { // find the owner of the process // get the desired principal to load String keytabFilename = conf.get(JTConfig.JT_KEYTAB_FILE); UserGroupInformation.setConfiguration(conf); if (keytabFilename != null) { String desiredUser = conf.get(JTConfig.JT_USER_NAME, System.getProperty("user.name")); UserGroupInformation.loginUserFromKeytab(desiredUser, keytabFilename); mrOwner = UserGroupInformation.getLoginUser(); } else {//from www .j av a2 s .c o m mrOwner = UserGroupInformation.getCurrentUser(); } supergroup = conf.get(MR_SUPERGROUP, "supergroup"); LOG.info("Starting jobtracker with owner as " + mrOwner.getShortUserName() + " and supergroup as " + supergroup); long secretKeyInterval = conf.getLong(MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_KEY, MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); long tokenMaxLifetime = conf.getLong(MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_KEY, MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); long tokenRenewInterval = conf.getLong(MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); secretManager = new DelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, DELEGATION_TOKEN_GC_INTERVAL); secretManager.startThreads(); // // Grab some static constants // NUM_HEARTBEATS_IN_SECOND = conf.getInt(JT_HEARTBEATS_IN_SECOND, DEFAULT_NUM_HEARTBEATS_IN_SECOND); if (NUM_HEARTBEATS_IN_SECOND < MIN_NUM_HEARTBEATS_IN_SECOND) { NUM_HEARTBEATS_IN_SECOND = DEFAULT_NUM_HEARTBEATS_IN_SECOND; } HEARTBEATS_SCALING_FACTOR = conf.getFloat(JT_HEARTBEATS_SCALING_FACTOR, DEFAULT_HEARTBEATS_SCALING_FACTOR); if (HEARTBEATS_SCALING_FACTOR < MIN_HEARTBEATS_SCALING_FACTOR) { HEARTBEATS_SCALING_FACTOR = DEFAULT_HEARTBEATS_SCALING_FACTOR; } // whether to dump or not every heartbeat message even when DEBUG is enabled dumpHeartbeat = conf.getBoolean(JT_HEARTBEATS_DUMP, false); // This is a directory of temporary submission files. We delete it // on startup, and can delete any files that we're done with this.conf = conf; JobConf jobConf = new JobConf(conf); // Set ports, start RPC servers, setup security policy etc. InetSocketAddress addr = getAddress(conf); this.localMachine = addr.getHostName(); this.port = addr.getPort(); int handlerCount = conf.getInt(JT_IPC_HANDLER_COUNT, 10); this.interTrackerServer = RPC.getServer(SkewTuneClientProtocol.class, this, addr.getHostName(), addr.getPort(), handlerCount, false, conf, secretManager); if (LOG.isDebugEnabled()) { Properties p = System.getProperties(); for (Iterator it = p.keySet().iterator(); it.hasNext();) { String key = (String) it.next(); String val = p.getProperty(key); LOG.debug("Property '" + key + "' is " + val); } } InetSocketAddress infoSocAddr = NetUtils .createSocketAddr(conf.get(JT_HTTP_ADDRESS, String.format("%s:0", this.localMachine))); String infoBindAddress = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); this.startTime = System.currentTimeMillis(); infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf); infoServer.setAttribute("job.tracker", this); infoServer.addServlet("jobcompletion", "/completion", JobCompletionServlet.class); infoServer.addServlet("taskspeculation", "/speculation", SpeculationEventServlet.class); infoServer.addServlet("skewreport", "/skew", SkewReportServlet.class); infoServer.addServlet("tasksplit", "/split/*", SplitTaskServlet.class); infoServer.addServlet("tasksplitV2", "/splitV2/*", SplitTaskV2Servlet.class); infoServer.start(); this.trackerIdentifier = jobtrackerIndentifier; // The rpc/web-server ports can be ephemeral ports... // ... ensure we have the correct info this.port = interTrackerServer.getListenerAddress().getPort(); this.conf.set(JT_IPC_ADDRESS, (this.localMachine + ":" + this.port)); LOG.info("JobTracker up at: " + this.port); this.infoPort = this.infoServer.getPort(); this.conf.set(JT_HTTP_ADDRESS, infoBindAddress + ":" + this.infoPort); LOG.info("JobTracker webserver: " + this.infoServer.getPort()); this.defaultNotificationUrl = String.format("http://%s:%d/completion?jobid=$jobId&status=$jobStatus", infoBindAddress, this.infoPort); LOG.info("JobTracker completion URI: " + defaultNotificationUrl); // this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?taskid=$taskId&remainTime=$taskRemainTime",infoBindAddress,this.infoPort); this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?jobid=$jobId", infoBindAddress, this.infoPort); LOG.info("JobTracker speculation event URI: " + defaultSpeculationEventUrl); this.defaultSkewReportUrl = String.format("http://%s:%d/skew", infoBindAddress, this.infoPort); LOG.info("JobTracker skew report event URI: " + defaultSkewReportUrl); this.trackerHttp = String.format("http://%s:%d", infoBindAddress, this.infoPort); while (!Thread.currentThread().isInterrupted()) { try { // if we haven't contacted the namenode go ahead and do it if (fs == null) { fs = mrOwner.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return FileSystem.get(conf); } }); } // clean up the system dir, which will only work if hdfs is out // of safe mode if (systemDir == null) { systemDir = new Path(getSystemDir()); } try { FileStatus systemDirStatus = fs.getFileStatus(systemDir); if (!systemDirStatus.getOwner().equals(mrOwner.getShortUserName())) { throw new AccessControlException( "The systemdir " + systemDir + " is not owned by " + mrOwner.getShortUserName()); } if (!systemDirStatus.getPermission().equals(SYSTEM_DIR_PERMISSION)) { LOG.warn("Incorrect permissions on " + systemDir + ". Setting it to " + SYSTEM_DIR_PERMISSION); fs.setPermission(systemDir, new FsPermission(SYSTEM_DIR_PERMISSION)); } else { break; } } catch (FileNotFoundException fnf) { } // ignore } catch (AccessControlException ace) { LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir + ") because of permissions."); LOG.warn("Manually delete the " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir + ") and then start the JobTracker."); LOG.warn("Bailing out ... "); throw ace; } catch (IOException ie) { LOG.info("problem cleaning system directory: " + systemDir, ie); } Thread.sleep(FS_ACCESS_RETRY_PERIOD); } if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } // initialize cluster variable cluster = new Cluster(this.conf); // now create a job client proxy jtClient = (ClientProtocol) RPC.getProxy(ClientProtocol.class, ClientProtocol.versionID, JobTracker.getAddress(conf), mrOwner, this.conf, NetUtils.getSocketFactory(conf, ClientProtocol.class)); new SpeculativeScheduler().start(); // initialize task event fetcher new TaskCompletionEventFetcher().start(); // Same with 'localDir' except it's always on the local disk. asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs()); asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR); // keep at least one asynchronous worker per CPU core int numProcs = Runtime.getRuntime().availableProcessors(); LOG.info("# of available processors = " + numProcs); int maxFactor = conf.getInt(JT_MAX_ASYNC_WORKER_FACTOR, 2); asyncWorkers = new ThreadPoolExecutor(numProcs, numProcs * maxFactor, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(true), new ThreadPoolExecutor.CallerRunsPolicy()); speculativeSplit = conf.getBoolean(JT_SPECULATIVE_SPLIT, false); }