List of usage examples for org.apache.hadoop.security Credentials getAllTokens
public Collection<Token<? extends TokenIdentifier>> getAllTokens()
From source file:oz.hadoop.yarn.test.cluster.InJvmContainerExecutor.java
License:Apache License
/** * * @param container// ww w. j a v a 2s . c o m * @param containerWorkDir * @return */ private UserGroupInformation buildUgiForContainerLaunching(Container container, final Path containerWorkDir) { UserGroupInformation ugi; try { ugi = UserGroupInformation.createRemoteUser(UserGroupInformation.getLoginUser().getUserName()); ugi.setAuthenticationMethod(AuthMethod.TOKEN); String filePath = new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE).toString(); Credentials credentials = Credentials.readTokenStorageFile(new File(filePath), this.getConf()); Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens(); for (Token<? extends TokenIdentifier> token : tokens) { ugi.addToken(token); } } catch (Exception e) { throw new IllegalArgumentException( "Failed to build UserGroupInformation to launch container " + container, e); } return ugi; }
From source file:uk.ac.gla.terrier.probos.cli.pbsdsh.java
License:Open Source License
@Override public int run(String[] args) throws Exception { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.debug("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.debug(token.toString());/*ww w . j av a2 s.com*/ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } Options options = new Options(); options.addOption("h", true, "Specify hostname."); CommandLineParser parser = new GnuParser(); CommandLine cmd = parser.parse(options, args); final String[] dibcommand = cmd.getArgs(); int jobId = getJobId(); if (jobId == -1) { System.err.println("PBS: PBS_JOBID not set"); return 1; } String[] hosts; PBSJobStatusDistributed d = (PBSJobStatusDistributed) c.getJobStatus(jobId, 5); if (d.getState() == '?') { System.err.println("PBS: Job " + jobId + " was lost"); return -1; } if (d.getState() != 'R') { System.err.println("PBS: Job " + jobId + " was not running"); return -1; } int[] ports; if (cmd.hasOption('h')) { hosts = cmd.getOptionValues('h'); ports = new int[hosts.length]; String[] tmpH = d.getHostnames(); int[] tmpP = d.getPorts(); TObjectIntHashMap<String> host2port = new TObjectIntHashMap<String>(tmpH.length); for (int i = 0; i < tmpH.length; i++) { host2port.put(tmpH[i], tmpP[i]); } int i = 0; for (String h : hosts) { if (!host2port.contains(h)) { throw new IllegalArgumentException("Host " + h + " is not a member of this distributed job"); } ports[i++] = host2port.get(h); } } else { hosts = d.getHostnames(); ports = d.getPorts(); } final String secret = d.getSecret(); if (secret == null) throw new IllegalArgumentException( "No secret found - pbsdsh called too early? " + Arrays.toString(d.getHostnames())); LOG.debug("To run on " + Arrays.toString(hosts)); final CyclicBarrier barrier = new CyclicBarrier(1 + hosts.length); int i = 0; for (final String h : hosts) { final int port = ports[i++]; new Thread() { @Override public void run() { try { if (connectToSister(h, port, secret, dibcommand) != 0) LOG.error("Could not connect"); } catch (Exception e) { LOG.error("Could not connect", e); } finally { try { barrier.await(); } catch (Exception e) { LOG.error("Barrier problem?"); } } } }.start(); } barrier.await(); return 0; }
From source file:uk.ac.gla.terrier.probos.job.ProbosJobService.java
License:Open Source License
@Override public int run(String[] args) throws Exception { final String secret = System.getenv("PBS_SECRET"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token.toString());/*from w w w. j av a 2 s .c om*/ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } SshServer sshd = getSSHServer(secret); sshd.start(); running.set(true); int jobId = Integer.parseInt(System.getenv("PBS_JOBID")); String hostPort = System.getenv("PBS_CONTROLLER"); String[] hostPortSplit = hostPort.split(":"); final String serverHostname = hostPortSplit[0]; final int serverPort = Integer.parseInt(hostPortSplit[1]); final InetSocketAddress server = new InetSocketAddress(serverHostname, serverPort); Configuration conf = this.getConf(); PBSInteractiveClient client = RPC.getProxy(PBSInteractiveClient.class, RPC.getProtocolVersion(PBSInteractiveClient.class), server, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf)); LOG.info("Sister for " + jobId + " started on " + sshd.getPort() + " with secret " + secret); informController(secret, sshd.getPort(), jobId, client); while (running.get()) { Thread.sleep(1000); } LOG.info("Ssh terminated by running variable"); sshd.stop(true); RPC.stopProxy(client); return 0; }
From source file:uk.ac.gla.terrier.probos.master.ProbosApplicationMasterServiceImpl.java
License:Open Source License
@SuppressWarnings("unchecked") public ProbosApplicationMasterServiceImpl(ApplicationMasterParameters parameters, Configuration _conf) throws Exception { super(parameters, _conf); LOG.info("Starting " + this.getClass().getSimpleName() + " on " + Utils.getHostname()); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); probosTokens = new ArrayList<Token<ProbosDelegationTokenIdentifier>>(); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing on " + Utils.getHostname() + " with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token.toString());/*from www. ja va2 s.com*/ if (token.getKind().equals(ProbosDelegationTokenIdentifier.KIND_NAME)) { probosTokens.add((Token<ProbosDelegationTokenIdentifier>) token); } } renewer = new ProbosTokenRenewer(); this.conf = _conf; StringWriter sw = new StringWriter(); Configuration.dumpConfiguration(conf, sw); //LOG.info("Master conf is " + sw.toString()); for (String k : REQUIRED_ENV) { if (System.getenv(k) == null) throw new IllegalArgumentException("Env " + k + " must be set"); } String hostPort = System.getenv("PBS_CONTROLLER"); String[] hostPortSplit = hostPort.split(":"); final String serverHostname = hostPortSplit[0]; final int port = Integer.parseInt(hostPortSplit[1]); final InetSocketAddress server = new InetSocketAddress(serverHostname, port); masterClient = RPC.getProxy(PBSMasterClient.class, RPC.getProtocolVersion(PBSMasterClient.class), server, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf)); controllerClient = PBSClientFactory.getPBSClient(); LOG.info("Connected to controller " + hostPort); jobId = Integer.parseInt(System.getenv("PBS_JOBID")); container = System.getenv("CONTAINER_ID"); masterClient.jobEvent(jobId, EventType.MASTER_START, container, null); final List<Entry<String, HttpServlet>> masterServlets = new ArrayList<>(); masterServlets.add(new MapEntry<String, HttpServlet>("/", new JobProgressServlet("./", masterServlets, controllerClient, this))); masterServlets.add(new MapEntry<String, HttpServlet>("/qstatjob", new QstatJobServlet("./qstatjob", masterServlets, controllerClient, this))); masterServlets.add(new MapEntry<String, HttpServlet>("/conf", new ConfServlet("./conf", masterServlets, controllerClient, this))); masterServlets.add(new MapEntry<String, HttpServlet>("/kittenconf", new KittenConfServlet("./kittenconf", masterServlets, controllerClient, this))); masterServlets.add(new MapEntry<String, HttpServlet>("/logs", new LogsServlet("./logs", masterServlets, controllerClient, this))); //0 means any random free port webServer = new WebServer("ProbosControllerHttp", masterServlets, 0); webServer.init(_conf); }
From source file:x10.x10rt.yarn.ApplicationMaster.java
License:Open Source License
private void setup() throws IOException, YarnException { LOG.info("Starting ApplicationMaster"); // Remove the AM->RM token so that containers cannot access it. Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token);/*from w ww . j ava 2s.c o m*/ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); UserGroupInformation appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, new RMCallbackHandler()); resourceManager.init(conf); resourceManager.start(); nodeManager = new NMClientAsyncImpl(new NMCallbackHandler(this)); nodeManager.init(conf); nodeManager.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); { int slash = appMasterHostname.indexOf('/'); if (slash != -1) appMasterHostname = appMasterHostname.substring(0, slash); } // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. // TODO: should we reject instead of modifying to fit? if (memoryPerPlaceInMb > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + memoryPerPlaceInMb + ", max=" + maxMem); memoryPerPlaceInMb = maxMem; } if (coresPerPlace > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } else if (coresPerPlace == 0) { LOG.info("Container virtual cores specified as auto (X10_NTHREADS=0)." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = initialNumPlaces - previousAMRunningContainers.size(); // open a local port for X10rt management, and register it with the selector launcherChannel = ServerSocketChannel.open(); //launcherChannel.bind(new InetSocketAddress(appMasterHostname, 0)); // bind to the visible network hostname and random port launcherChannel.bind(null); launcherChannel.configureBlocking(false); appMasterPort = launcherChannel.socket().getLocalPort(); launcherChannel.register(selector, SelectionKey.OP_ACCEPT); numRequestedContainers.set(initialNumPlaces); // Send request for containers to RM for (int i = 0; i < numTotalContainersToRequest; ++i) { Resource capability = Resource.newInstance(memoryPerPlaceInMb, coresPerPlace); ContainerRequest request = new ContainerRequest(capability, null, null, Priority.newInstance(0)); LOG.info("Requested container ask: " + request.toString()); resourceManager.addContainerRequest(request); pendingRequests.add(request); } }
From source file:yarnkit.utils.YarnUtils.java
License:Apache License
public static void removeToken(@Nonnull final Credentials credentials, @Nonnull final Text tokenKindToRemove) { Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(tokenKindToRemove)) { iter.remove();/* w ww . jav a 2 s. c om*/ } } }