List of usage examples for org.apache.hadoop.fs FileSystem closeAllForUGI
public static void closeAllForUGI(UserGroupInformation ugi) throws IOException
From source file:joshelser.TUGIAssumingProcessor.java
License:Apache License
@Override public boolean process(final TProtocol inProt, final TProtocol outProt) throws TException { TTransport trans = inProt.getTransport(); if (!(trans instanceof TSaslServerTransport)) { throw new TException("Unexpected non-SASL transport " + trans.getClass()); }/* w w w . j a v a2 s.co m*/ TSaslServerTransport saslTrans = (TSaslServerTransport) trans; SaslServer saslServer = saslTrans.getSaslServer(); String authId = saslServer.getAuthorizationID(); String endUser = authId; UserGroupInformation clientUgi = null; try { clientUgi = UserGroupInformation.createProxyUser(endUser, UserGroupInformation.getLoginUser()); final String remoteUser = clientUgi.getShortUserName(); log.debug("Executing action as {}", remoteUser); return clientUgi.doAs(new PrivilegedExceptionAction<Boolean>() { @Override public Boolean run() { try { return wrapped.process(inProt, outProt); } catch (TException te) { throw new RuntimeException(te); } } }); } catch (RuntimeException rte) { if (rte.getCause() instanceof TException) { log.error("Failed to invoke wrapped processor", rte.getCause()); throw (TException) rte.getCause(); } throw rte; } catch (InterruptedException | IOException e) { log.error("Failed to invoke wrapped processor", e); throw new RuntimeException(e); } finally { if (clientUgi != null) { try { FileSystem.closeAllForUGI(clientUgi); } catch (IOException exception) { log.error("Could not clean up file-system handles for UGI: {}", clientUgi, exception); } } } }
From source file:org.apache.hawq.pxf.service.servlet.SecurityServletFilter.java
License:Apache License
/** * If user impersonation is configured, examines the request for the presense of the expected security headers * and create a proxy user to execute further request chain. Responds with an HTTP error if the header is missing * or the chain processing throws an exception. * * @param request http request/*from w w w.ja v a2 s .c o m*/ * @param response http response * @param chain filter chain */ @Override public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain) throws IOException, ServletException { if (SecureLogin.isUserImpersonationEnabled()) { // retrieve user header and make sure header is present and is not empty final String user = ((HttpServletRequest) request).getHeader(USER_HEADER); if (user == null) { throw new IllegalArgumentException(MISSING_HEADER_ERROR); } else if (user.trim().isEmpty()) { throw new IllegalArgumentException(EMPTY_HEADER_ERROR); } // TODO refresh Kerberos token when security is enabled // prepare pivileged action to run on behalf of proxy user PrivilegedExceptionAction<Boolean> action = new PrivilegedExceptionAction<Boolean>() { @Override public Boolean run() throws IOException, ServletException { LOG.debug("Performing request chain call for proxy user = " + user); chain.doFilter(request, response); return true; } }; // create proxy user UGI from the UGI of the logged in user and execute the servlet chain as that user UserGroupInformation proxyUGI = null; try { LOG.debug("Creating proxy user = " + user); proxyUGI = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); proxyUGI.doAs(action); } catch (UndeclaredThrowableException ute) { // unwrap the real exception thrown by the action throw new ServletException(ute.getCause()); } catch (InterruptedException ie) { throw new ServletException(ie); } finally { try { if (proxyUGI != null) { LOG.debug("Closing FileSystem for proxy user = " + proxyUGI.getUserName()); FileSystem.closeAllForUGI(proxyUGI); } } catch (Throwable t) { LOG.warn("Error closing FileSystem for proxy user = " + proxyUGI.getUserName()); } } } else { // no user impersonation is configured chain.doFilter(request, response); } }
From source file:org.apache.hawq.pxf.service.UGIProvider.java
License:Apache License
/** * Wrapper for {@link FileSystem}.closeAllForUGI method. * @param ugi the {@link UserGroupInformation} whose filesystem resources we want to free. * @throws IOException// www .j av a 2s. c om */ void destroy(UserGroupInformation ugi) throws IOException { FileSystem.closeAllForUGI(ugi); }
From source file:org.apache.hive.service.cli.session.HiveSessionImplwithUGI.java
License:Apache License
/** * Close the file systems for the session and remove it from the FileSystem cache. * Cancel the session's delegation token and close the metastore connection *//*from ww w. j ava 2 s. c o m*/ @Override public void close() throws HiveSQLException { try { acquire(true); cancelDelegationToken(); } finally { try { super.close(); } finally { try { FileSystem.closeAllForUGI(sessionUgi); } catch (IOException ioe) { throw new HiveSQLException("Could not clean up file-system handles for UGI: " + sessionUgi, ioe); } } } }
From source file:org.apache.oozie.service.UserGroupInformationService.java
License:Apache License
@Override public void destroy() { for (UserGroupInformation ugi : cache.values()) { try {//from w w w. j av a 2s . co m FileSystem.closeAllForUGI(ugi); } catch (IOException ioe) { XLog.getLog(this.getClass()) .warn("Exception occurred while closing filesystems for " + ugi.getUserName(), ioe); } } cache.clear(); }
From source file:org.apache.tez.dag.app.DAGAppMaster.java
License:Apache License
@VisibleForTesting protected synchronized void handle(DAGAppMasterEvent event) { switch (event.getType()) { case SCHEDULING_SERVICE_ERROR: // Scheduling error - probably an issue with the communication with the RM // In this scenario, the AM should shutdown. Expectation is that the RM // will restart a new AM attempt. // Should not kill the current running DAG to ensure that on restart, we // can recover it and continue. DAGAppMasterEventSchedulingServiceError schedulingServiceErrorEvent = (DAGAppMasterEventSchedulingServiceError) event; state = DAGAppMasterState.ERROR; LOG.info("Error in the TaskScheduler. Shutting down.", schedulingServiceErrorEvent.getThrowable()); shutdownHandler.shutdown();/*from w ww.ja v a 2 s.c o m*/ break; case INTERNAL_ERROR: state = DAGAppMasterState.ERROR; if (currentDAG != null) { _updateLoggers(currentDAG, "_post"); // notify dag to finish which will send the DAG_FINISHED event LOG.info("Internal Error. Notifying dags to finish."); sendEvent(new DAGEvent(currentDAG.getID(), DAGEventType.INTERNAL_ERROR)); } else { LOG.info("Internal Error. Finishing directly as no dag is active."); this.taskSchedulerEventHandler.setShouldUnregisterFlag(); shutdownHandler.shutdown(); } break; case DAG_FINISHED: DAGAppMasterEventDAGFinished finishEvt = (DAGAppMasterEventDAGFinished) event; if (!isSession) { LOG.info("Not a session, AM will unregister as DAG has completed"); this.taskSchedulerEventHandler.setShouldUnregisterFlag(); _updateLoggers(currentDAG, "_post"); setStateOnDAGCompletion(); LOG.info("Shutting down on completion of dag:" + finishEvt.getDAGId().toString()); shutdownHandler.shutdown(); } else { LOG.info("DAG completed, dagId=" + finishEvt.getDAGId().toString() + ", dagState=" + finishEvt.getDAGState()); lastDAGCompletionTime = clock.getTime(); _updateLoggers(currentDAG, "_post"); if (this.historyEventHandler.hasRecoveryFailed()) { LOG.warn("Recovery had a fatal error, shutting down session after" + " DAG completion"); sessionStopped.set(true); } switch (finishEvt.getDAGState()) { case SUCCEEDED: if (!currentDAG.getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { successfulDAGs.incrementAndGet(); } break; case FAILED: if (!currentDAG.getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { failedDAGs.incrementAndGet(); } break; case KILLED: if (!currentDAG.getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { killedDAGs.incrementAndGet(); } break; case ERROR: if (!currentDAG.getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { failedDAGs.incrementAndGet(); } // This is a pass-through. Kill the AM if DAG state is ERROR. default: LOG.fatal("Received a DAG Finished Event with state=" + finishEvt.getDAGState() + ". Error. Shutting down."); state = DAGAppMasterState.ERROR; this.taskSchedulerEventHandler.setShouldUnregisterFlag(); shutdownHandler.shutdown(); break; } if (!state.equals(DAGAppMasterState.ERROR)) { if (!sessionStopped.get()) { LOG.info("Waiting for next DAG to be submitted."); this.taskSchedulerEventHandler.dagCompleted(); state = DAGAppMasterState.IDLE; } else { LOG.info("Session shutting down now."); this.taskSchedulerEventHandler.setShouldUnregisterFlag(); if (this.historyEventHandler.hasRecoveryFailed()) { state = DAGAppMasterState.FAILED; } else { state = DAGAppMasterState.SUCCEEDED; } shutdownHandler.shutdown(); } } } //close all fs related caches try { FileSystem.closeAllForUGI(context.getCurrentDAG().getDagUGI()); } catch (IOException e) { LOG.warn("Error occurred when trying to close FileSystem for userName " + context.getCurrentDAG().getDagUGI().getUserName(), e); } break; case AM_REBOOT: LOG.info("Received an AM_REBOOT signal"); this.state = DAGAppMasterState.KILLED; shutdownHandler.shutdown(true); break; default: throw new TezUncheckedException("AppMaster: No handler for event type: " + event.getType()); } }
From source file:org.apache.tez.runtime.task.TezChild.java
License:Apache License
public ContainerExecutionResult run() throws IOException, InterruptedException, TezException { ContainerContext containerContext = new ContainerContext(containerIdString); ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, getTaskMaxSleepTime);/* ww w. j a v a2 s . co m*/ taskReporter = new TaskReporter(umbilical, amHeartbeatInterval, sendCounterInterval, maxEventsToGet, heartbeatCounter, containerIdString); UserGroupInformation childUGI = null; while (!executor.isTerminated()) { if (taskCount > 0) { TezUtilsInternal.updateLoggers(""); } ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter); boolean error = false; ContainerTask containerTask = null; try { containerTask = getTaskFuture.get(); } catch (ExecutionException e) { error = true; Throwable cause = e.getCause(); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, cause, "Execution Exception while fetching new work: " + e.getMessage()); } catch (InterruptedException e) { error = true; LOG.info("Interrupted while waiting for new work"); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.INTERRUPTED, e, "Interrupted while waiting for new work"); } finally { if (error) { shutdown(); } } if (containerTask.shouldDie()) { LOG.info("ContainerTask returned shouldDie=true, Exiting"); shutdown(); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM"); } else { String loggerAddend = containerTask.getTaskSpec().getTaskAttemptID().toString(); taskCount++; TezUtilsInternal.updateLoggers(loggerAddend); FileSystem.clearStatistics(); childUGI = handleNewTaskCredentials(containerTask, childUGI); handleNewTaskLocalResources(containerTask); cleanupOnTaskChanged(containerTask); // Execute the Actual Task TezTaskRunner taskRunner = new TezTaskRunner(defaultConf, childUGI, localDirs, containerTask.getTaskSpec(), umbilical, appAttemptNumber, serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memAvailable); boolean shouldDie; try { shouldDie = !taskRunner.run(); if (shouldDie) { LOG.info("Got a shouldDie notification via hearbeats. Shutting down"); shutdown(); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM"); } } catch (IOException e) { handleError(e); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, e, "TaskExecutionFailure: " + e.getMessage()); } catch (TezException e) { handleError(e); return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, e, "TaskExecutionFailure: " + e.getMessage()); } finally { FileSystem.closeAllForUGI(childUGI); } } } return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, null); }