List of usage examples for org.apache.hadoop.security.authorize ServiceAuthorizationManager refresh
public void refresh(Configuration conf, PolicyProvider provider)
From source file:backup.datanode.DataNodeBackupServicePlugin.java
License:Apache License
@Override public void start(Object service) { DataNode datanode = (DataNode) service; Configuration conf = getConf(); RPC.setProtocolEngine(conf, DataNodeBackupRPC.class, WritableRpcEngine.class); // This object is created here so that it's lifecycle follows the datanode try {/*from www .ja va 2s.c o m*/ backupProcessor = SingletonManager.getManager(DataNodeBackupProcessor.class).getInstance(datanode, () -> new DataNodeBackupProcessor(conf, datanode)); restoreProcessor = SingletonManager.getManager(DataNodeRestoreProcessor.class).getInstance(datanode, () -> new DataNodeRestoreProcessor(conf, datanode)); DataNodeBackupRPCImpl backupRPCImpl = new DataNodeBackupRPCImpl(backupProcessor, restoreProcessor); InetSocketAddress listenerAddress = datanode.ipcServer.getListenerAddress(); int ipcPort = listenerAddress.getPort(); String bindAddress = listenerAddress.getAddress().getHostAddress(); int port = conf.getInt(DFS_BACKUP_DATANODE_RPC_PORT_KEY, DFS_BACKUP_DATANODE_RPC_PORT_DEFAULT); if (port == 0) { port = ipcPort + 1; } server = new RPC.Builder(conf).setBindAddress(bindAddress).setPort(port).setInstance(backupRPCImpl) .setProtocol(DataNodeBackupRPC.class).build(); ServiceAuthorizationManager serviceAuthorizationManager = server.getServiceAuthorizationManager(); serviceAuthorizationManager.refresh(conf, new BackupPolicyProvider()); server.start(); LOG.info("DataNode Backup RPC listening on {}", port); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:backup.namenode.NameNodeBackupServicePlugin.java
License:Apache License
@Override public void start(Object service) { UserGroupInformation ugi;//from w w w . j av a2 s . c o m try { ugi = UserGroupInformation.getCurrentUser(); LOG.info("Starting NameNodeBackupServicePlugin with ugi {}", ugi); } catch (IOException e) { throw new RuntimeException(e); } Configuration conf = getConf(); NameNode namenode = (NameNode) service; BlockManager blockManager = namenode.getNamesystem().getBlockManager(); // This object is created here so that it's lifecycle follows the namenode try { restoreProcessor = SingletonManager.getManager(NameNodeRestoreProcessor.class).getInstance(namenode, () -> new NameNodeRestoreProcessor(getConf(), namenode, ugi)); LOG.info("NameNode Backup plugin setup using UGI {}", ugi); NameNodeBackupRPCImpl backupRPCImpl = new NameNodeBackupRPCImpl(blockManager); InetSocketAddress listenerAddress = namenode.getServiceRpcAddress(); int ipcPort = listenerAddress.getPort(); String bindAddress = listenerAddress.getAddress().getHostAddress(); int port = conf.getInt(DFS_BACKUP_NAMENODE_RPC_PORT_KEY, DFS_BACKUP_NAMENODE_RPC_PORT_DEFAULT); if (port == 0) { port = ipcPort + 1; } server = new RPC.Builder(conf).setBindAddress(bindAddress).setPort(port).setInstance(backupRPCImpl) .setProtocol(NameNodeBackupRPC.class).build(); ServiceAuthorizationManager serviceAuthorizationManager = server.getServiceAuthorizationManager(); serviceAuthorizationManager.refresh(conf, new BackupPolicyProvider()); server.start(); LOG.info("NameNode Backup RPC listening on {}", port); int httpPort = getConf().getInt(DFS_BACKUP_NAMENODE_HTTP_PORT_KEY, DFS_BACKUP_NAMENODE_HTTP_PORT_DEFAULT); if (httpPort != 0) { ClassLoader classLoader = getClassLoader(); if (classLoader != null) { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { BackupWebService<Stats> stats = getBackupWebService(ugi, blockManager, restoreProcessor); // Have to setup classloader in thread context to get the static // files in the http server tp be setup correctly. Thread.currentThread().setContextClassLoader(classLoader); Class<?> backupStatusServerClass = classLoader.loadClass(BACKUP_WEB_BACKUP_WEB_SERVER); Object server = DuckTypeUtil.newInstance(backupStatusServerClass, new Class[] { Integer.TYPE, BackupWebService.class }, new Object[] { httpPort, stats }); httpServer = DuckTypeUtil.wrap(HttpServer.class, server); httpServer.start(); LOG.info("NameNode Backup HTTP listening on {}", httpPort); } finally { Thread.currentThread().setContextClassLoader(contextClassLoader); } } else { LOG.info("NameNode Backup HTTP classes not found."); } } } catch (Exception e) { throw new RuntimeException(e); } }
From source file:common.DataNode.java
License:Apache License
/** * This method starts the data node with the specified conf. * /*from w ww.j ava 2 s . c o m*/ * @param conf - the configuration * if conf's CONFIG_PROPERTY_SIMULATED property is set * then a simulated storage based data node is created. * * @param dataDirs - only for a non-simulated storage data node * @throws IOException */ void startDataNode(Configuration conf, AbstractList<File> dataDirs, DatanodeProtocol namenode) throws IOException { // use configured nameserver & interface to get local hostname if (conf.get(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY) != null) { machineName = conf.get(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY); } if (machineName == null) { machineName = DNS.getDefaultHost(conf.get("dfs.datanode.dns.interface", "default"), conf.get("dfs.datanode.dns.nameserver", "default")); } this.nameNodeAddr = NameNode.getAddress(conf); this.socketTimeout = conf.getInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, HdfsConstants.READ_TIMEOUT); this.socketWriteTimeout = conf.getInt("dfs.datanode.socket.write.timeout", HdfsConstants.WRITE_TIMEOUT); /* Based on results on different platforms, we might need set the default * to false on some of them. */ this.transferToAllowed = conf.getBoolean("dfs.datanode.transferTo.allowed", true); this.writePacketSize = conf.getInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT); InetSocketAddress socAddr = NetUtils.createSocketAddr(conf.get("dfs.datanode.address", "0.0.0.0:50010")); int tmpPort = socAddr.getPort(); storage = new DataStorage(); // construct registration this.dnRegistration = new DatanodeRegistration(machineName + ":" + tmpPort); // connect to name node this.namenode = namenode; // get version and id info from the name-node NamespaceInfo nsInfo = handshake(); StartupOption startOpt = getStartupOption(conf); assert startOpt != null : "Startup option must be set."; boolean simulatedFSDataset = conf.getBoolean("dfs.datanode.simulateddatastorage", false); if (simulatedFSDataset) { setNewStorageID(dnRegistration); dnRegistration.storageInfo.layoutVersion = FSConstants.LAYOUT_VERSION; dnRegistration.storageInfo.namespaceID = nsInfo.namespaceID; // it would have been better to pass storage as a parameter to // constructor below - need to augment ReflectionUtils used below. conf.set(DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY, dnRegistration.getStorageID()); try { //Equivalent of following (can't do because Simulated is in test dir) // this.data = new SimulatedFSDataset(conf); this.data = (FSDatasetInterface) ReflectionUtils.newInstance( Class.forName("org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset"), conf); } catch (ClassNotFoundException e) { throw new IOException(StringUtils.stringifyException(e)); } } else { // real storage // read storage info, lock data dirs and transition fs state if necessary storage.recoverTransitionRead(nsInfo, dataDirs, startOpt); // adjust this.dnRegistration.setStorageInfo(storage); // initialize data node internal structure this.data = new FSDataset(storage, conf); } // find free port ServerSocket ss = (socketWriteTimeout > 0) ? ServerSocketChannel.open().socket() : new ServerSocket(); Server.bind(ss, socAddr, 0); ss.setReceiveBufferSize(DEFAULT_DATA_SOCKET_SIZE); // adjust machine name with the actual port tmpPort = ss.getLocalPort(); selfAddr = new InetSocketAddress(ss.getInetAddress().getHostAddress(), tmpPort); this.dnRegistration.setName(machineName + ":" + tmpPort); LOG.info("Opened info server at " + tmpPort); this.threadGroup = new ThreadGroup("dataXceiverServer"); this.dataXceiverServer = new Daemon(threadGroup, new DataXceiverServer(ss, conf, this)); this.threadGroup.setDaemon(true); // auto destroy when empty this.blockReportInterval = conf.getLong("dfs.blockreport.intervalMsec", BLOCKREPORT_INTERVAL); this.initialBlockReportDelay = conf.getLong("dfs.blockreport.initialDelay", BLOCKREPORT_INITIAL_DELAY) * 1000L; if (this.initialBlockReportDelay >= blockReportInterval) { this.initialBlockReportDelay = 0; LOG.info("dfs.blockreport.initialDelay is greater than " + "dfs.blockreport.intervalMsec." + " Setting initial delay to 0 msec:"); } this.heartBeatInterval = conf.getLong("dfs.heartbeat.interval", HEARTBEAT_INTERVAL) * 1000L; //initialize periodic block scanner String reason = null; if (conf.getInt("dfs.datanode.scan.period.hours", 0) < 0) { reason = "verification is turned off by configuration"; } else if (!(data instanceof FSDataset)) { reason = "verifcation is supported only with FSDataset"; } if (reason == null) { blockScanner = new DataBlockScanner(this, (FSDataset) data, conf); } else { LOG.info("Periodic Block Verification is disabled because " + reason + "."); } //create a servlet to serve full-file content InetSocketAddress infoSocAddr = NetUtils .createSocketAddr(conf.get("dfs.datanode.http.address", "0.0.0.0:50075")); String infoHost = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); this.infoServer = new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0, conf); if (conf.getBoolean("dfs.https.enable", false)) { boolean needClientAuth = conf.getBoolean(DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY, DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT); InetSocketAddress secInfoSocAddr = NetUtils .createSocketAddr(conf.get("dfs.datanode.https.address", infoHost + ":" + 0)); Configuration sslConf = new HdfsConfiguration(false); sslConf.addResource(conf.get("dfs.https.server.keystore.resource", "ssl-server.xml")); this.infoServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth); } this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class); this.infoServer.addInternalServlet(null, "/getFileChecksum/*", FileChecksumServlets.GetServlet.class); this.infoServer.setAttribute("datanode.blockScanner", blockScanner); this.infoServer.setAttribute("datanode.conf", conf); this.infoServer.addServlet(null, "/blockScannerReport", DataBlockScanner.Servlet.class); this.infoServer.start(); // adjust info port this.dnRegistration.setInfoPort(this.infoServer.getPort()); myMetrics = new DataNodeMetrics(conf, dnRegistration.getName()); // set service-level authorization security policy if (conf.getBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) { ServiceAuthorizationManager.refresh(conf, new HDFSPolicyProvider()); } //init ipc server InetSocketAddress ipcAddr = NetUtils.createSocketAddr(conf.get("dfs.datanode.ipc.address")); ipcServer = RPC.getServer(DataNode.class, this, ipcAddr.getHostName(), ipcAddr.getPort(), conf.getInt("dfs.datanode.handler.count", 3), false, conf); ipcServer.start(); dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort()); LOG.info("dnRegistration = " + dnRegistration); plugins = conf.getInstances("dfs.datanode.plugins", ServicePlugin.class); for (ServicePlugin p : plugins) { try { p.start(this); LOG.info("Started plug-in " + p); } catch (Throwable t) { LOG.warn("ServicePlugin " + p + " could not be started", t); } } }
From source file:common.NameNode.java
License:Apache License
/** * Initialize name-node./*from w ww .j ava 2 s . c o m*/ * * @param conf the configuration */ protected void initialize(Configuration conf) throws IOException { InetSocketAddress socAddr = getRpcServerAddress(conf); int handlerCount = conf.getInt("dfs.namenode.handler.count", 10); // set service-level authorization security policy if (serviceAuthEnabled = conf.getBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) { ServiceAuthorizationManager.refresh(conf, new HDFSPolicyProvider()); } NameNode.initMetrics(conf, this.getRole()); loadNamesystem(conf); // create rpc server this.server = RPC.getServer(NamenodeProtocols.class, this, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf, namesystem.getDelegationTokenSecretManager()); // The rpc-server port can be ephemeral... ensure we have the correct info this.rpcAddress = this.server.getListenerAddress(); setRpcServerAddress(conf); activate(conf); LOG.info(getRole() + " up at: " + rpcAddress); }
From source file:common.NameNode.java
License:Apache License
@Override public void refreshServiceAcl() throws IOException { if (!serviceAuthEnabled) { throw new AuthorizationException("Service Level Authorization not enabled!"); }/*w w w. ja v a2 s.com*/ ServiceAuthorizationManager.refresh(new Configuration(), new HDFSPolicyProvider()); }