List of usage examples for org.apache.hadoop.security UserGroupInformation doAs
@InterfaceAudience.Public @InterfaceStability.Evolving public <T> T doAs(PrivilegedExceptionAction<T> action) throws IOException, InterruptedException
From source file:org.apache.tez.runtime.task.TezChild.java
License:Apache License
public TezChild(Configuration conf, String host, int port, String containerIdentifier, String tokenIdentifier, int appAttemptNumber, String workingDir, String[] localDirs, Map<String, String> serviceProviderEnvMap, ObjectRegistryImpl objectRegistry, String pid, ExecutionContext executionContext, Credentials credentials, long memAvailable, String user) throws IOException, InterruptedException { this.defaultConf = conf; this.containerIdString = containerIdentifier; this.appAttemptNumber = appAttemptNumber; this.localDirs = localDirs; this.serviceProviderEnvMap = serviceProviderEnvMap; this.workingDir = workingDir; this.pid = pid; this.executionContext = executionContext; this.credentials = credentials; this.memAvailable = memAvailable; this.user = user; getTaskMaxSleepTime = defaultConf.getInt(TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX, TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT); amHeartbeatInterval = defaultConf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT); sendCounterInterval = defaultConf.getLong(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT); maxEventsToGet = defaultConf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT, TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT); ExecutorService executor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TezChild").build()); this.executor = MoreExecutors.listeningDecorator(executor); this.objectRegistry = objectRegistry; if (LOG.isDebugEnabled()) { LOG.debug("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.debug(token);/*from w ww . jav a 2 s . co m*/ } } this.isLocal = defaultConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT); UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(tokenIdentifier); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, TezCommonUtils.convertJobTokenToBytes(jobToken)); if (!isLocal) { final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port); SecurityUtil.setTokenService(jobToken, address); taskOwner.addToken(jobToken); umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() { @Override public TezTaskUmbilicalProtocol run() throws Exception { return RPC.getProxy(TezTaskUmbilicalProtocol.class, TezTaskUmbilicalProtocol.versionID, address, defaultConf); } }); } }
From source file:org.apache.twill.filesystem.LocationTestBase.java
License:Apache License
@Test public void testHomeLocation() throws Exception { LocationFactory locationFactory = createLocationFactory("/"); // Without UGI, the home location should be the same as the user Assert.assertEquals(System.getProperty("user.name"), locationFactory.getHomeLocation().getName()); // With UGI, the home location should be based on the UGI current user UserGroupInformation ugi = UserGroupInformation.createRemoteUser(System.getProperty("user.name") + "1"); locationFactory = ugi.doAs(new PrivilegedExceptionAction<LocationFactory>() { @Override//from www.j a va 2s . com public LocationFactory run() throws Exception { return createLocationFactory("/"); } }); Assert.assertEquals(ugi.getUserName(), locationFactory.getHomeLocation().getName()); }
From source file:org.apache.zeppelin.jdbc.JDBCInterpreter.java
License:Apache License
public Connection getConnection(String propertyKey, InterpreterContext interpreterContext) throws ClassNotFoundException, SQLException, InterpreterException, IOException { final String user = interpreterContext.getAuthenticationInfo().getUser(); Connection connection;/*from w ww . ja va2 s . c o m*/ if (propertyKey == null || basePropretiesMap.get(propertyKey) == null) { return null; } JDBCUserConfigurations jdbcUserConfigurations = getJDBCConfiguration(user); setUserProperty(propertyKey, interpreterContext); final Properties properties = jdbcUserConfigurations.getPropertyMap(propertyKey); final String url = properties.getProperty(URL_KEY); if (isEmpty(getProperty("zeppelin.jdbc.auth.type"))) { connection = getConnectionFromPool(url, user, propertyKey, properties); } else { UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(getProperties()); final String connectionUrl = appendProxyUserToURL(url, user, propertyKey); JDBCSecurityImpl.createSecureConfiguration(getProperties(), authType); switch (authType) { case KERBEROS: if (user == null || "false".equalsIgnoreCase(getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) { connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties); } else { if (basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) { connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties); } else { UserGroupInformation ugi = null; try { ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getCurrentUser()); } catch (Exception e) { logger.error("Error in getCurrentUser", e); throw new InterpreterException("Error in getCurrentUser", e); } final String poolKey = propertyKey; try { connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() { @Override public Connection run() throws Exception { return getConnectionFromPool(connectionUrl, user, poolKey, properties); } }); } catch (Exception e) { logger.error("Error in doAs", e); throw new InterpreterException("Error in doAs", e); } } } break; default: connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties); } } return connection; }
From source file:org.apache.zeppelin.scalding.ScaldingInterpreter.java
License:Apache License
@Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { String user = contextInterpreter.getAuthenticationInfo().getUser(); logger.info("Running Scalding command: user: {} cmd: '{}'", user, cmd); if (interpreter == null) { logger.error("interpreter == null, open may not have been called because max.open.instances reached"); return new InterpreterResult(Code.ERROR, "interpreter == null\n" + "open may not have been called because max.open.instances reached"); }// w w w.j a v a2s. co m if (cmd == null || cmd.trim().length() == 0) { return new InterpreterResult(Code.SUCCESS); } InterpreterResult interpreterResult = new InterpreterResult(Code.ERROR); if (getProperty(ARGS_STRING).contains("hdfs")) { UserGroupInformation ugi = null; try { ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); } catch (IOException e) { logger.error("Error creating UserGroupInformation", e); return new InterpreterResult(Code.ERROR, e.getMessage()); } try { // Make variables final to avoid "local variable is accessed from within inner class; // needs to be declared final" exception in JDK7 final String cmd1 = cmd; final InterpreterContext contextInterpreter1 = contextInterpreter; PrivilegedExceptionAction<InterpreterResult> action = new PrivilegedExceptionAction<InterpreterResult>() { public InterpreterResult run() throws Exception { return interpret(cmd1.split("\n"), contextInterpreter1); } }; interpreterResult = ugi.doAs(action); } catch (Exception e) { logger.error("Error running command with ugi.doAs", e); return new InterpreterResult(Code.ERROR, e.getMessage()); } } else { interpreterResult = interpret(cmd.split("\n"), contextInterpreter); } return interpreterResult; }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.appmaster.ApplicationMaster.java
License:Apache License
@Override public int run(String[] args) throws Exception { // Set our own configuration (ToolRunner only sets it prior to calling // run())/*from w w w . j a v a 2 s . co m*/ conf = getConf(); // Our own RM Handler ResourceManagerHandler rmHandler = new ResourceManagerHandler(conf, appAttemptId); // Connect rmHandler.getAMResourceManager(); // Register try { rmHandler.registerApplicationMaster(masterHost, masterPort); } catch (YarnRemoteException ex) { LOG.error("Error encountered while trying to register application master", ex); return ReturnCode.MASTER_ERROR.getCode(); } // Get file splits, configuration, etc. Set<ConfigurationTuple> configTuples; try { configTuples = getConfigurationTuples(); } catch (IOException ex) { LOG.error("Error encountered while trying to generate configurations", ex); return ReturnCode.MASTER_ERROR.getCode(); } // Needed for our master service later Map<WorkerId, StartupConfiguration> startupConf = getMasterStartupConfiguration(configTuples); // Initial containers we want, based off of the file splits List<ResourceRequest> requestedContainers = getRequestedContainersList(configTuples, rmHandler); List<ContainerId> releasedContainers = new ArrayList<>(); // Send an initial allocation request List<Container> allocatedContainers = new ArrayList<>(); try { int needed = configTuples.size(); int got = 0; int maxAttempts = Integer.parseInt(props.getProperty(ConfigFields.APP_ALLOCATION_MAX_ATTEMPTS, "10")); int attempts = 0; List<Container> acquiredContainers; while (got < needed && attempts < maxAttempts) { LOG.info("Requesting containers" + ", got=" + got + ", needed=" + needed + ", attempts=" + attempts + ", maxAttempts=" + maxAttempts); acquiredContainers = rmHandler.allocateRequest(requestedContainers, releasedContainers) .getAllocatedContainers(); got += acquiredContainers.size(); attempts++; allocatedContainers.addAll(acquiredContainers); acquiredContainers.clear(); LOG.info("Got allocation response, allocatedContainers=" + acquiredContainers.size()); Thread.sleep(2500); } } catch (YarnRemoteException ex) { LOG.error("Encountered an error while trying to allocate containers", ex); return ReturnCode.MASTER_ERROR.getCode(); } final int numContainers = configTuples.size(); /* * * * TODO: fix this so we try N times to get enough containers! * * * * */ // Make sure we got all our containers, or else bail if (allocatedContainers.size() < numContainers) { LOG.info("Unable to get required number of containers, will not continue" + ", needed=" + numContainers + ", allocated=" + allocatedContainers.size()); requestedContainers.clear(); // We don't want new containers! // Add containers into released list for (Container c : allocatedContainers) { releasedContainers.add(c.getId()); } // Release containers try { rmHandler.allocateRequest(requestedContainers, releasedContainers); } catch (YarnRemoteException ex) { LOG.warn("Encountered an error while trying to release unwanted containers", ex); } // Notify our handlers that we got a problem rmHandler.finishApplication("Unable to allocate containers, needed " + numContainers + ", but got " + allocatedContainers.size(), FinalApplicationStatus.FAILED); // bail return ReturnCode.MASTER_ERROR.getCode(); } // Launch our worker process, as we now expect workers to actally do // something LOG.info("Starting master service"); ApplicationMasterService<T> masterService = new ApplicationMasterService<>(masterAddr, startupConf, masterComputable, masterUpdateable, appConfig, conf); ExecutorService executor = Executors.newSingleThreadExecutor(); Future<Integer> masterThread = executor.submit(masterService); // We got the number of containers we wanted, let's launch them LOG.info("Launching child containers"); List<Thread> launchThreads = launchContainers(configTuples, allocatedContainers); // Use an empty list for heartbeat purposes requestedContainers.clear(); // Some local counters. Do we really need Atomic? AtomicInteger numCompletedContainers = new AtomicInteger(); AtomicInteger numFailedContainers = new AtomicInteger(); LOG.info("Waiting for containers to complete..."); // Go into run-loop waiting for containers to finish, also our heartbeat while (numCompletedContainers.get() < numContainers) { // Don't pound the RM try { Thread.sleep(2000); } catch (InterruptedException ex) { LOG.warn("Interrupted while waiting on completed containers", ex); return ReturnCode.MASTER_ERROR.getCode(); } // Heartbeat, effectively List<ContainerStatus> completedContainers; try { completedContainers = rmHandler.allocateRequest(requestedContainers, releasedContainers) .getCompletedContainersStatuses(); } catch (YarnRemoteException ex) { LOG.warn("Encountered an error while trying to heartbeat to resource manager", ex); continue; // Nothing to report, probably an error / endless loop } for (ContainerStatus cs : completedContainers) { int exitCode = cs.getExitStatus(); if (exitCode != 0) { numCompletedContainers.incrementAndGet(); numFailedContainers.incrementAndGet(); masterService.fail(); executor.shutdown(); // Force kill our application, fail fast? LOG.info("At least one container failed with a non-zero exit code (" + exitCode + "); killing application"); rmHandler.finishApplication( "Failing, due to at least container coming back with an non-zero exit code.", FinalApplicationStatus.KILLED); return -10; } else { numCompletedContainers.incrementAndGet(); } } } // All containers have completed // Wait for launch threads to complete (this shouldn't really happen) LOG.info("Containers completed"); for (Thread launchThread : launchThreads) { try { launchThread.join(1000); } catch (InterruptedException ex) { LOG.warn("Interrupted while waiting for Launcher threads to complete", ex); } } // Ensure that our master service has completed as well if (!masterThread.isDone()) { masterService.fail(); } int masterExit = masterThread.get(); LOG.info("Master service completed with exitCode=" + masterExit); executor.shutdown(); if (masterExit == 0) { String impersonatedUser = System.getenv("USER"); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(impersonatedUser); //UserGroupInformation.createProxyUser(impersonatedUser, UserGroupInformation.getLoginUser()); ugi.doAs(new PrivilegedExceptionAction<Void>() { public Void run() { Path out = new Path(props.getProperty(ConfigFields.APP_OUTPUT_PATH)); FileSystem fs; try { fs = out.getFileSystem(conf); FSDataOutputStream fos = fs.create(out); LOG.info("Writing master results to " + out.toString()); masterComputable.complete(fos); fos.flush(); fos.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return null; //FileSystem fs = FileSystem.get(conf); //fs.mkdir( out ); } }); /* LOG.info( "Here we would try to write to " + out.toString() ); LOG.info( "As current user: " + UserGroupInformation.getCurrentUser().getShortUserName() ); LOG.info( "As login user: " + UserGroupInformation.getLoginUser().getShortUserName() ); LOG.info( "Env Var User: " + System.getenv("USER") ); */ //LOG.info( "Ideally we'd be user: " + this.props.getProperty( ) ); // for (Map.Entry<String, String> entry : this.conf) { // LOG.info("ApplicationMaster->Conf: " + entry.getKey() + " = " + entry.getValue()); // } } else { LOG.warn("Not writing master results, as the master came back with errors!"); } // Application finished ReturnCode rc = (numFailedContainers.get() == 0) ? ReturnCode.OK : ReturnCode.CONTAINER_ERROR; try { if (numFailedContainers.get() == 0) { rmHandler.finishApplication("Completed successfully", FinalApplicationStatus.SUCCEEDED); } else { String diag = "Completed with " + numFailedContainers.get() + " failed containers"; rmHandler.finishApplication(diag, FinalApplicationStatus.FAILED); } } catch (YarnRemoteException ex) { LOG.warn("Encountered an error while trying to send final status to resource manager", ex); } return rc.getCode(); }
From source file:org.elasticsearch.hadoop.yarn.rpc.YarnRpc.java
License:Apache License
public void start() { // handle security if (UserGroupInformation.isSecurityEnabled()) { UserGroupInformation.setConfiguration(cfg); }/*from w ww . ja v a2 s . co m*/ try { endpoint = resolveEndpoint(cfg); } catch (IOException ex) { throw new EsYarnException("Cannot resolve endpoint", ex); } UserGroupInformation ugi = null; try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException ex) { throw new EsYarnException("Cannot get current user", ex); } // create proxy proxy = ugi.doAs(new PrivilegedAction<P>() { @SuppressWarnings("unchecked") @Override public P run() { return (P) YarnRPC.create(cfg).getProxy(protocolType, endpoint, cfg); } }); }
From source file:org.kitesdk.spring.hbase.example.service.WebPageSnapshotService.java
License:Apache License
/** * Take a snapshot of an URL. This WebPageSnapshot is stored in HBase. Returns * the WebPageSnapshotMeta// w w w . j ava 2s . c om * * If the URL is a redirect, the snapshot is stored under the final URL * destination. A WebPageRedirectModel is stored in the redirect table so when * fetching snapshots, we can follow the proper redirect path. * * @param url The URL to take a snapshot of * @param contentKey The key used to store the content * @param user The user taking a snapshot * @return The WebPageSnapshotMeta for the page that we snapshotted. * @throws IOException */ public WebPageSnapshotMeta takeSnapshot(final String url, final String contentKey, final String user) throws IOException { WebPageSnapshotMeta meta = null; UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); try { meta = ugi.doAs(new PrivilegedExceptionAction<WebPageSnapshotMeta>() { @Override public WebPageSnapshotMeta run() throws Exception { WebPageSnapshotModel webPageSnapshotModel = fetchWebPage(url, contentKey); if (!webPageSnapshotModel.getUrl().equals(url)) { // Url is different, so must have redirected. Store the redirect model WebPageRedirectModel redirectModel = WebPageRedirectModel.newBuilder().setUrl(url) .setDestinationUrl(webPageSnapshotModel.getUrl()).build(); webPageRedirectModels(user).put(redirectModel); } else { // If redirect exists, remove it since this URL no longer redirects Key key = new Key.Builder(webPageRedirectModels(user)).add("url", url).build(); WebPageRedirectModel redirectModel = webPageRedirectModels(user).get(key); if (redirectModel != null) { webPageRedirectModels(user).delete(key); } } webPageSnapshotModels(user).put(webPageSnapshotModel); return conversionService.convert(webPageSnapshotModel, WebPageSnapshotMeta.class); } }); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); if (meta == null) { throw new IOException("Interrupted trying to save the snapshot", ex); } } return meta; }
From source file:org.kitesdk.spring.hbase.example.service.WebPageSnapshotService.java
License:Apache License
/** * Get the epoch timestamps for every snapshot time of an URL in HBase. * * @param url The URL of the page to get snapshot timestamps for * @return The list of timestamps//from w ww. j ava 2 s .c om */ public List<Long> getSnapshotTimestamps(String url, final String user) throws IOException { List<Long> snapshotTimestamps = null; final String normalizedUrl = normalizeUrl(url, user); LOG.error("Getting snapshot timestamps: url = {}, user = {}, normalized url = {}", new Object[] { url, user, normalizedUrl }); UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); snapshotTimestamps = ugi.doAs(new PrivilegedAction<List<Long>>() { @Override public List<Long> run() { List<Long> snapshotTimestamps = new ArrayList<Long>(); DatasetReader<WebPageSnapshotModel> reader = null; try { reader = webPageSnapshotModels(user).from("url", normalizedUrl).from("fetchedAtRevTs", 0L) .to("url", normalizedUrl).to("fetchedAtRevTs", Long.MAX_VALUE).newReader(); while (reader.hasNext()) { snapshotTimestamps.add(reader.next().getFetchedAt()); } } finally { if (reader != null) { reader.close(); } } return snapshotTimestamps; } }); return snapshotTimestamps; }
From source file:org.kitesdk.spring.hbase.example.service.WebPageSnapshotService.java
License:Apache License
/** * Get the most recent WebPageSnapshotModel from HBase * * @param url The URL to get the snapshotted page from HBase * @return The WebPageSnapshotModel, or null if there are no fetches for this * URL//from w ww . jav a 2 s.c o m */ private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url, final String user) throws IOException { WebPageSnapshotModel snapshot = null; final String normalizedUrl = normalizeUrl(url, user); UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); LOG.error("Created proxy user " + ugi.getShortUserName() + " ugi: " + ugi); snapshot = ugi.doAs(new PrivilegedAction<WebPageSnapshotModel>() { @Override public WebPageSnapshotModel run() { DatasetReader<WebPageSnapshotModel> reader = null; try { // we don't know the exact timestamp in the key, but we know since keys // are in timestamp descending order that the first row for an URL will be // the most recent. reader = webPageSnapshotModels(user).from("url", normalizedUrl).from("fetchedAtRevTs", 0L) .to("url", normalizedUrl).to("fetchedAtRevTs", Long.MAX_VALUE).newReader(); if (reader.hasNext()) { return reader.next(); } else { return null; } } finally { if (reader != null) { reader.close(); } } } }); return snapshot; }
From source file:org.kitesdk.spring.hbase.example.service.WebPageSnapshotService.java
License:Apache License
/** * Get the WebPageSnapshotModel from HBase * * @param url The URL of the WebPageSnapshotModel * @param ts The snapshot timestamp of the WebPageSnapshotModel * @return The WebPageSnapshotModel, or null if there is no snapshot for the * URL at this timestamp./*from ww w . j a v a 2 s . c o m*/ */ private WebPageSnapshotModel getWebPageSnapshot(String url, final long ts, final String user) throws IOException { WebPageSnapshotModel snapshot = null; final String normalizedUrl = normalizeUrl(url, user); UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); snapshot = ugi.doAs(new PrivilegedAction<WebPageSnapshotModel>() { @Override public WebPageSnapshotModel run() { Key key = new Key.Builder(webPageSnapshotModels(user)).add("url", normalizedUrl) .add("fetchedAtRevTs", Long.MAX_VALUE - ts).build(); return webPageSnapshotModels(user).get(key); } }); return snapshot; }