List of usage examples for org.apache.hadoop.net NetUtils createSocketAddr
public static InetSocketAddress createSocketAddr(String target)
From source file:azkaban.security.HadoopSecurityManager_H_2_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final Props props, final Logger logger) throws HadoopSecurityManagerException { final String userToProxy = props.getString(USER_TO_PROXY); logger.info("Getting hadoop tokens based on props for " + userToProxy); final Credentials cred = new Credentials(); if (props.getBoolean(OBTAIN_HCAT_TOKEN, false)) { try {//w w w . java2 s .co m // first we fetch and save the default hcat token. logger.info("Pre-fetching default Hive MetaStore token from hive"); HiveConf hiveConf = new HiveConf(); Token<DelegationTokenIdentifier> hcatToken = fetchHcatToken(userToProxy, hiveConf, null, logger); cred.addToken(hcatToken.getService(), hcatToken); // check and see if user specified the extra hcat locations we need to // look at and fetch token. final List<String> extraHcatLocations = props.getStringList(EXTRA_HCAT_LOCATION); if (Collections.EMPTY_LIST != extraHcatLocations) { logger.info("Need to pre-fetch extra metaStore tokens from hive."); // start to process the user inputs. for (String thriftUrl : extraHcatLocations) { logger.info("Pre-fetching metaStore token from : " + thriftUrl); hiveConf = new HiveConf(); hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, thriftUrl); hcatToken = fetchHcatToken(userToProxy, hiveConf, thriftUrl, logger); cred.addToken(hcatToken.getService(), hcatToken); } } } catch (Throwable t) { String message = "Failed to get hive metastore token." + t.getMessage() + t.getCause(); logger.error(message, t); throw new HadoopSecurityManagerException(message); } } if (props.getBoolean(OBTAIN_JOBHISTORYSERVER_TOKEN, false)) { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); logger.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); logger.info("Pre-fetching JH token from job history server"); Token<?> jhsdt = null; try { jhsdt = getDelegationTokenFromHS(hsProxy); } catch (Exception e) { logger.error("Failed to fetch JH token", e); throw new HadoopSecurityManagerException("Failed to fetch JH token for " + userToProxy); } if (jhsdt == null) { logger.error("getDelegationTokenFromHS() returned null"); throw new HadoopSecurityManagerException("Unable to fetch JH token for " + userToProxy); } logger.info("Created JH token: " + jhsdt.toString()); logger.info("Token kind: " + jhsdt.getKind()); logger.info("Token id: " + jhsdt.getIdentifier()); logger.info("Token service: " + jhsdt.getService()); cred.addToken(jhsdt.getService(), jhsdt); } try { getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { logger.info("Here is the props for " + OBTAIN_NAMENODE_TOKEN + ": " + props.getBoolean(OBTAIN_NAMENODE_TOKEN)); if (props.getBoolean(OBTAIN_NAMENODE_TOKEN, false)) { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the // conf logger.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs .getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException( "Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); // getting additional name nodes tokens String otherNamenodes = props.get(OTHER_NAMENODES_TO_GET_TOKEN); if ((otherNamenodes != null) && (otherNamenodes.length() > 0)) { logger.info(OTHER_NAMENODES_TO_GET_TOKEN + ": '" + otherNamenodes + "'"); String[] nameNodeArr = otherNamenodes.split(","); Path[] ps = new Path[nameNodeArr.length]; for (int i = 0; i < ps.length; i++) { ps[i] = new Path(nameNodeArr[i].trim()); } TokenCache.obtainTokensForNamenodes(cred, ps, conf); logger.info("Successfully fetched tokens for: " + otherNamenodes); } else { logger.info(OTHER_NAMENODES_TO_GET_TOKEN + " was not configured"); } } if (props.getBoolean(OBTAIN_JOBTRACKER_TOKEN, false)) { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); logger.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient .getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { logger.error("Failed to fetch JT token"); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } } }); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); cred.writeTokenStorageToStream(dos); } finally { if (dos != null) { try { dos.close(); } catch (Throwable t) { // best effort logger.error("encountered exception while closing DataOutputStream of the tokenFile", t); } } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } catch (Exception e) { throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause(), e); } catch (Throwable t) { throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + t.getMessage() + t.getCause(), t); } }
From source file:azkaban.security.HadoopSecurityManager_H_2_0.java
License:Apache License
private Text getMRTokenRenewerInternal(JobConf jobConf) throws IOException { // Taken from Oozie ////w w w.j a va2s . c om // Getting renewer correctly for JT principal also though JT in hadoop // 1.x does not have // support for renewing/cancelling tokens String servicePrincipal = jobConf.get(RM_PRINCIPAL, jobConf.get(JT_PRINCIPAL)); Text renewer; if (servicePrincipal != null) { String target = jobConf.get(HADOOP_YARN_RM, jobConf.get(HADOOP_JOB_TRACKER_2)); if (target == null) { target = jobConf.get(HADOOP_JOB_TRACKER); } String addr = NetUtils.createSocketAddr(target).getHostName(); renewer = new Text(SecurityUtil.getServerPrincipal(servicePrincipal, addr)); } else { // No security renewer = DEFAULT_RENEWER; } return renewer; }
From source file:co.cask.cdap.operations.hdfs.HDFSInfo.java
License:Apache License
@Nullable private URL rpcToHttpAddress(URI rpcURI) throws MalformedURLException { String host = rpcURI.getHost(); if (host == null) { return null; }/*from w w w. ja v a 2 s .c om*/ boolean httpsEnabled = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT); String namenodeWebAddress = httpsEnabled ? conf.get(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT) : conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT); InetSocketAddress socketAddress = NetUtils.createSocketAddr(namenodeWebAddress); int namenodeWebPort = socketAddress.getPort(); String protocol = httpsEnabled ? "https" : "http"; return new URL(protocol, host, namenodeWebPort, ""); }
From source file:com.chinamobile.bcbsp.bspcontroller.BSPController.java
License:Apache License
/** * get BSPcontroller socket address./*from w ww . ja va2s . c o m*/ * @param conf * BSP conf to get the address. * @return * socket address of controller. */ public static InetSocketAddress getAddress(Configuration conf) { String bspControllerStr = conf.get(Constants.BC_BSP_CONTROLLER_ADDRESS); return NetUtils.createSocketAddr(bspControllerStr); }
From source file:com.chinamobile.bcbsp.client.BSPJobClient.java
License:Apache License
/** * Ensure fresh jobSubmitClient.// w w w. j a va2 s . com */ public void ensureFreshjJobSubmitClient() { try { if (bspzk != null) { Stat s = null; int count = 0; int max = 3; while (s == null && count < max) { count++; Thread.sleep(500); s = bspzk.exists(Constants.BSPCONTROLLER_LEADER, true); } if (s != null) { String controllerAddr = getData(Constants.BSPCONTROLLER_LEADER); InetSocketAddress newAddr = NetUtils.createSocketAddr(controllerAddr); if (this.bspControllerAddr == null || !this.bspControllerAddr.equals(newAddr)) { this.bspControllerAddr = newAddr; // establish connection to new BspController again conf.set("ipc.client.connect.max.retries", "0"); this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy(JobSubmissionProtocol.class, BSPRPCProtocolVersion.versionID, bspControllerAddr, conf, NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); LOG.info("Now connected to " + this.bspControllerAddr.toString()); } } } } catch (Exception e) { // LOG.warn("lost connection to " + this.bspControllerAddr.toString()); // LOG.error("[ensureFreshjJobSubmitClient]", e); throw new RuntimeException("lost connection to bspControllerAddr ", e); } }
From source file:com.chinamobile.bcbsp.workermanager.WorkerManager.java
License:Apache License
/** * Initialize workerManager.//w ww . j av a 2s . c o m */ @SuppressWarnings("static-access") public synchronized void initialize() throws IOException { if (this.conf.get(Constants.BC_BSP_WORKERMANAGER_RPC_HOST) != null) { this.workerManagerName = conf.get(Constants.BC_BSP_WORKERMANAGER_RPC_HOST); } if (this.workerManagerName == null) { this.workerManagerName = DNS.getDefaultHost(conf.get("bsp.dns.interface", "default"), conf.get("bsp.dns.nameserver", "default")); } // check local disk checkLocalDirs(conf.getStrings(Constants.BC_BSP_LOCAL_DIRECTORY)); deleteLocalFiles("workerManager"); this.workerFaultList = new ArrayList<Fault>(); this.reportStaffStatusList = new ArrayList<StaffStatus>(); this.runningStaffs = new ConcurrentHashMap<StaffAttemptID, StaffInProgress>(); this.finishedStaffs = new ConcurrentHashMap<StaffAttemptID, StaffInProgress>(); this.runningJobs = new ConcurrentHashMap<BSPJobID, RunningJob>(); this.finishedJobs = new ConcurrentHashMap<BSPJobID, RunningJob>(); this.runningJobtoWorkerAgent = new ConcurrentHashMap<BSPJobID, WorkerAgentForJob>(); this.reprotStaffsMap = new ConcurrentHashMap<StaffAttemptID, StaffInProgress>(); this.conf.set(Constants.BC_BSP_WORKERAGENT_HOST, this.workerManagerName); this.conf.set(Constants.BC_BSP_WORKERMANAGER_RPC_HOST, this.workerManagerName); this.maxStaffsCount = conf.getInt(Constants.BC_BSP_WORKERMANAGER_MAXSTAFFS, 1); WorkerManager.HEART_BEAT_INTERVAL = conf.getInt(Constants.HEART_BEAT_INTERVAL, 1000); LOG.info("The max number of staffs is : " + this.maxStaffsCount); int rpcPort = -1; String rpcAddr = null; if (!this.initialized) { rpcAddr = conf.get(Constants.BC_BSP_WORKERMANAGER_RPC_HOST, Constants.DEFAULT_BC_BSP_WORKERMANAGER_RPC_HOST); rpcPort = conf.getInt(Constants.BC_BSP_WORKERMANAGER_RPC_PORT, 5000); if (-1 == rpcPort || null == rpcAddr) { throw new IllegalArgumentException("Error rpc address " + rpcAddr + " port" + rpcPort); } this.workerServer = RPC.getServer(this, rpcAddr, rpcPort, conf); this.workerServer.start(); this.rpcServer = rpcAddr + ":" + rpcPort; LOG.info("Worker rpc server --> " + rpcServer); } String address = conf.get(Constants.BC_BSP_WORKERMANAGER_REPORT_ADDRESS); InetSocketAddress socAddr = NetUtils.createSocketAddr(address); String bindAddress = socAddr.getHostName(); int tmpPort = socAddr.getPort(); // RPC initialization this.staffReportServer = RPC.getServer(this, bindAddress, tmpPort, 10, false, this.conf); this.staffReportServer.start(); // http server InetAddress addr = InetAddress.getLocalHost(); String ipSlave = addr.getHostAddress().toString(); winfoPort = conf.getInt("bcbsp.http.winfoserver.port", 40027); winfoServer = new HttpServer("bcbsp", ipSlave, winfoPort, true, conf); winfoServer.setAttribute("WorkerManager", this); LOG.info("prot: 40027"); LOG.info("ljn test : controllerClient before start "); winfoServer.start(); LOG.info("server has started"); LOG.info("ljn test : controllerClient before register "); // get the assigned address this.staffReportAddress = staffReportServer.getListenerAddress(); LOG.info("WorkerManager up at: " + this.staffReportAddress); DistributedCache.purgeCache(this.conf); LOG.info("ljn test : DistributedCache "); LOG.info("ljn test : bspControllerAddr " + bspControllerAddr); LOG.info("ljn test : BSPRPCProtocolVersion.versionID " + BSPRPCProtocolVersion.versionID); // establish the communication link to bsp master try { this.controllerClient = (ControllerProtocol) RPC.waitForProxy(ControllerProtocol.class, BSPRPCProtocolVersion.versionID, bspControllerAddr, conf); } catch (Exception e) { throw new RuntimeException(e); } LOG.info("ljn test : controllerClient controllerClient start "); // establish the communication link to standby bsp master if ("ha".equals(conf.get(Constants.BC_BSP_HA_FLAG, ""))) { this.standbyControllerClient = (ControllerProtocol) RPC.waitForProxy(ControllerProtocol.class, BSPRPCProtocolVersion.versionID, this.standbyControllerAddr, conf); } LOG.info("bspControllerAddr = " + bspControllerAddr + " standbyControllerAddr = " + standbyControllerAddr); LOG.info("ljn test : controllerClient before register "); // enroll in bsp controller if (-1 == rpcPort || null == rpcAddr) { throw new IllegalArgumentException("Error rpc address " + rpcAddr + " port" + rpcPort); } LOG.info("ljn test : controllerClient before lsManager.start "); this.lsManager.start(); LOG.info("ljn test : controllerClient before lsManager.start over"); workerMangerStatus = new WorkerManagerStatus(workerManagerName, cloneAndResetRunningStaffStatuses(), maxStaffsCount, currentStaffsCount, finishedStaffsCount, failures, this.rpcServer, workerFaultList); this.workerMangerStatus.setHost(bindAddress); this.workerMangerStatus.setHttpPort(this.staffReportAddress.toString()); this.workerMangerStatus.setLocalIp(ipSlave); LOG.info("ljn test : controllerClient before register "); if (!this.controllerClient.register(workerMangerStatus)) { LOG.error("There is a problem in establishing communication" + " link with BSPController"); throw new IOException("There is a problem in establishing" + " communication link with BSPController."); } else { LOG.info("have registed to bsp master"); } if ("ha".equals(conf.get(Constants.BC_BSP_HA_FLAG, ""))) { if (!this.standbyControllerClient.register(workerMangerStatus)) { LOG.error("There is a problem in establishing communication" + " link with BSPController"); throw new IOException( "There is a problem in establishing" + " communication link with BSPController."); } else { LOG.info("have registed to standby bsp master"); } } this.running = true; this.initialized = true; LOG.info("ljn test : controllerClient after register "); }
From source file:com.chinamobile.bcbsp.workermanager.WorkerManager.java
License:Apache License
@Override public void process(WatchedEvent event) { LOG.info("now in process"); LOG.info("event type is " + event.getType()); try {//from w w w.ja v a 2 s .c om if (event.getType().toString().equals("NodeDeleted")) { LOG.info("in NodeDeleted"); if (bspzk != null) { if (bspzk.equaltoStat(Constants.BSPCONTROLLER_STANDBY_LEADER, true)) { String standControllerAddr = getData(Constants.BSPCONTROLLER_LEADER); InetSocketAddress newStandbyAddr = NetUtils.createSocketAddr(standControllerAddr); if (!this.standbyControllerAddr.equals(newStandbyAddr)) { this.bspControllerAddr = this.standbyControllerAddr; this.standbyControllerAddr = newStandbyAddr; this.controllerClient = (ControllerProtocol) RPC.getProxy(ControllerProtocol.class, BSPRPCProtocolVersion.versionID, this.bspControllerAddr, conf); this.standbyControllerClient = (ControllerProtocol) RPC.getProxy( ControllerProtocol.class, BSPRPCProtocolVersion.versionID, this.standbyControllerAddr, conf); } LOG.info("now the active is " + this.bspControllerAddr.toString() + "and the standby is " + this.standbyControllerAddr.toString()); } } } else if (event.getType().toString().equals("NodeDataChanged")) { // watch the standby bspzk.exists(Constants.BSPCONTROLLER_STANDBY_LEADER, true); // establish the communication link to standby bsp master this.standbyControllerClient = (ControllerProtocol) RPC.getProxy(ControllerProtocol.class, BSPRPCProtocolVersion.versionID, this.standbyControllerAddr, conf); LOG.info("bspControllerAddr = " + bspControllerAddr + " standbyControllerAddr = " + standbyControllerAddr); if (!this.standbyControllerClient.register(workerMangerStatus)) { LOG.error("There is a problem in establishing communication" + " link with BSPController"); throw new IOException( "There is a problem in establishing" + " communication link with BSPController."); } else { LOG.info("have registed to standby bsp master"); } } } catch (Exception e) { LOG.error("problem happened when register to standby controller " + e.toString()); } }
From source file:com.chinamobile.bcbsp.workermanager.WorkerManager.java
License:Apache License
/** * Get the address of BspController whose role is Active *//*from w w w . ja v a 2s . com*/ public void choseActiveControllerAddress() { String zkAddress = conf.get(Constants.ZOOKEEPER_QUORUM) + ":" + conf.getInt(Constants.ZOOKEPER_CLIENT_PORT, Constants.DEFAULT_ZOOKEPER_CLIENT_PORT); try { this.bspzk = new BSPZookeeperImpl(zkAddress, Constants.SESSION_TIME_OUT, this); if (bspzk != null) { if (bspzk.equaltoStat(Constants.BSPCONTROLLER_LEADER, true)) { String controllerAddr = getData(Constants.BSPCONTROLLER_LEADER); LOG.info("active controller Address is " + controllerAddr); this.bspControllerAddr = NetUtils.createSocketAddr(controllerAddr); } else { LOG.error("could not get the active BspController's " + "address,please restart the System"); } // s = zk.exists(Constants.BSPCONTROLLER_STANDBY_LEADER, true); if (bspzk.equaltoStat(Constants.BSPCONTROLLER_STANDBY_LEADER, true)) { String standControllerAddr = getData(Constants.BSPCONTROLLER_STANDBY_LEADER); LOG.info("standby controller Address is " + standControllerAddr); this.standbyControllerAddr = NetUtils.createSocketAddr(standControllerAddr); } else { LOG.info("could not get the standby BspController's address," + "please restart the standby System"); } } } catch (Exception e) { LOG.error("could not get the active BspController's address," + "please restart the System:" + e.getMessage()); } }
From source file:com.cloudera.kitten.appmaster.service.ContainerManagerConnectionFactoryImpl.java
License:Open Source License
@Override public synchronized ContainerManager connect(Container container) { NodeId nodeId = container.getNodeId(); String containerIpPort = String.format("%s:%d", nodeId.getHost(), nodeId.getPort()); if (!containerManagers.containsKey(containerIpPort)) { LOG.info("Connecting to ContainerManager at: " + containerIpPort); InetSocketAddress addr = NetUtils.createSocketAddr(containerIpPort); ContainerManager cm = (ContainerManager) rpc.getProxy(ContainerManager.class, addr, conf); containerManagers.put(containerIpPort, cm); return cm; }/*from w w w.j a v a 2s . com*/ return containerManagers.get(containerIpPort); }
From source file:com.cloudera.kitten.appmaster.service.ResourceManagerConnectionFactory.java
License:Open Source License
@Override public AMRMProtocol connect() { YarnConfiguration yarnConf = new YarnConfiguration(conf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); LOG.info("Connecting to ResourceManager at " + rmAddress); return ((AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf)); }