List of usage examples for org.apache.hadoop.security.token Token toString
@Override
public String toString()
From source file:azkaban.security.commons.SecurityUtils.java
License:Apache License
public static synchronized void prefetchToken(final File tokenFile, final Props p, final Logger logger) throws InterruptedException, IOException { final Configuration conf = new Configuration(); logger.info("Getting proxy user for " + p.getString(TO_PROXY)); logger.info("Getting proxy user for " + p.toString()); getProxiedUser(p.toProperties(), logger, conf).doAs(new PrivilegedExceptionAction<Void>() { @Override/*from w ww .j a va 2 s.c om*/ public Void run() throws Exception { getToken(p); return null; } private void getToken(Props p) throws InterruptedException, IOException { String shouldPrefetch = p.getString(OBTAIN_BINARY_TOKEN); if (shouldPrefetch != null && shouldPrefetch.equals("true")) { logger.info("Pre-fetching token"); logger.info("Pre-fetching fs token"); FileSystem fs = FileSystem.get(conf); Token<?> fsToken = fs.getDelegationToken(p.getString("user.to.proxy")); logger.info("Created token: " + fsToken.toString()); Job job = new Job(conf, "totally phony, extremely fake, not real job"); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching job token: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("hi")); logger.info("Created token: " + mrdt.toString()); job.getCredentials().addToken(new Text("howdy"), mrdt); job.getCredentials().addToken(fsToken.getService(), fsToken); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); job.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } logger.info("Loading hadoop tokens into " + tokenFile.getAbsolutePath()); p.put("HadoopTokenFileLoc", tokenFile.getAbsolutePath()); } else { logger.info("Not pre-fetching token"); } } }); }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final String userToProxy, final Logger logger) throws HadoopSecurityManagerException { logger.info("Getting hadoop tokens for " + userToProxy); try {/* w w w.j ava 2s . com*/ getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the conf logger.info("Getting DFS token from " + fs.getCanonicalServiceName() + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(userToProxy); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException("Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching JT token: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token")); if (mrdt == null) { logger.error("Failed to fetch JT token for "); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); jc.getCredentials().addToken(mrdt.getService(), mrdt); jc.getCredentials().addToken(fsToken.getService(), fsToken); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); jc.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } }); } catch (Exception e) { e.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause()); } }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
@Override public void cancelTokens(File tokenFile, String userToProxy, Logger logger) throws HadoopSecurityManagerException { // nntoken/* w w w . j av a 2 s . c om*/ Credentials cred = null; try { cred = Credentials.readTokenStorageFile(new Path(tokenFile.toURI()), new Configuration()); for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) { logger.info("Got token: " + t.toString()); logger.info("Token kind: " + t.getKind()); logger.info("Token id: " + new String(t.getIdentifier())); logger.info("Token service: " + t.getService()); if (t.getKind().equals(new Text("HIVE_DELEGATION_TOKEN"))) { logger.info("Cancelling hive token " + new String(t.getIdentifier())); cancelHiveToken(t, userToProxy); } else if (t.getKind().equals(new Text("MAPREDUCE_DELEGATION_TOKEN"))) { logger.info("Cancelling mr job tracker token " + new String(t.getIdentifier())); cancelMRJobTrackerToken(t, userToProxy); } else if (t.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) { logger.info("Cancelling namenode token " + new String(t.getIdentifier())); cancelNameNodeToken(t, userToProxy); } else { logger.info("unknown token type " + t.getKind()); } } } catch (Exception e) { e.printStackTrace(); } }
From source file:azkaban.security.HadoopSecurityManager_H_1_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final Props props, final Logger logger) throws HadoopSecurityManagerException { final String userToProxy = props.getString(USER_TO_PROXY); logger.info("Getting hadoop tokens for " + userToProxy); final Credentials cred = new Credentials(); if (props.getBoolean(OBTAIN_HCAT_TOKEN, false)) { try {/*from w w w. j av a 2 s .c o m*/ logger.info("Pre-fetching Hive MetaStore token from hive"); HiveConf hiveConf = new HiveConf(); logger.info("HiveConf.ConfVars.METASTOREURIS.varname " + hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname)); logger.info("HIVE_METASTORE_SASL_ENABLED " + hiveConf.get(HIVE_METASTORE_SASL_ENABLED)); logger.info("HIVE_METASTORE_KERBEROS_PRINCIPAL " + hiveConf.get(HIVE_METASTORE_KERBEROS_PRINCIPAL)); logger.info("HIVE_METASTORE_LOCAL " + hiveConf.get(HIVE_METASTORE_LOCAL)); HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(hiveConf); String hcatTokenStr = hiveClient.getDelegationToken(userToProxy, UserGroupInformation.getLoginUser().getShortUserName()); Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>(); hcatToken.decodeFromUrlString(hcatTokenStr); logger.info("Created hive metastore token: " + hcatTokenStr); logger.info("Token kind: " + hcatToken.getKind()); logger.info("Token id: " + hcatToken.getIdentifier()); logger.info("Token service: " + hcatToken.getService()); cred.addToken(hcatToken.getService(), hcatToken); } catch (Exception e) { e.printStackTrace(); logger.error("Failed to get hive metastore token." + e.getMessage() + e.getCause()); } catch (Throwable t) { t.printStackTrace(); logger.error("Failed to get hive metastore token." + t.getMessage() + t.getCause()); } } try { getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { logger.info("Here is the props for " + OBTAIN_NAMENODE_TOKEN + ": " + props.getBoolean(OBTAIN_NAMENODE_TOKEN)); if (props.getBoolean(OBTAIN_NAMENODE_TOKEN, false)) { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the // conf logger.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(userToProxy); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException( "Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); } if (props.getBoolean(OBTAIN_JOBTRACKER_TOKEN, false)) { JobClient jobClient = new JobClient(new JobConf()); logger.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token")); if (mrdt == null) { logger.error("Failed to fetch JT token"); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } } }); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); cred.writeTokenStorageToStream(dos); } finally { if (dos != null) { dos.close(); } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } catch (Exception e) { e.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause()); } catch (Throwable t) { t.printStackTrace(); throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + t.getMessage() + t.getCause()); } }
From source file:azkaban.security.HadoopSecurityManager_H_2_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final String userToProxy, final Logger logger) throws HadoopSecurityManagerException { logger.info("Getting hadoop tokens for " + userToProxy); try {/*from w w w. j ava 2s .c om*/ getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the conf logger.info("Getting DFS token from " + fs.getCanonicalServiceName() + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(userToProxy); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException("Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); JobConf jc = new JobConf(conf); JobClient jobClient = new JobClient(jc); logger.info("Pre-fetching JT token: Got new JobClient: " + jc); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(new Text("mr token")); if (mrdt == null) { logger.error("Failed to fetch JT token for "); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); jc.getCredentials().addToken(mrdt.getService(), mrdt); jc.getCredentials().addToken(fsToken.getService(), fsToken); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); jc.getCredentials().writeTokenStorageToStream(dos); } finally { if (dos != null) { try { dos.close(); } catch (Throwable t) { // best effort logger.error( "encountered exception while closing DataOutputStream of the tokenFile", t); } } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } }); } catch (Exception e) { throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause()); } }
From source file:azkaban.security.HadoopSecurityManager_H_2_0.java
License:Apache License
@Override public void cancelTokens(File tokenFile, String userToProxy, Logger logger) throws HadoopSecurityManagerException { // nntoken//from w ww .j a va2s . co m Credentials cred = null; try { cred = Credentials.readTokenStorageFile(new Path(tokenFile.toURI()), new Configuration()); for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) { logger.info("Got token: " + t.toString()); logger.info("Token kind: " + t.getKind()); logger.info("Token id: " + new String(t.getIdentifier())); logger.info("Token service: " + t.getService()); if (t.getKind().equals(new Text("HIVE_DELEGATION_TOKEN"))) { logger.info("Cancelling hive token " + new String(t.getIdentifier())); cancelHiveToken(t, userToProxy); } else if (t.getKind().equals(new Text("RM_DELEGATION_TOKEN"))) { logger.info("Cancelling mr job tracker token " + new String(t.getIdentifier())); // cancelMRJobTrackerToken(t, userToProxy); } else if (t.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) { logger.info("Cancelling namenode token " + new String(t.getIdentifier())); // cancelNameNodeToken(t, userToProxy); } else if (t.getKind().equals(new Text("MR_DELEGATION_TOKEN"))) { logger.info("Cancelling jobhistoryserver mr token " + new String(t.getIdentifier())); // cancelJhsToken(t, userToProxy); } else { logger.info("unknown token type " + t.getKind()); } } } catch (Exception e) { throw new HadoopSecurityManagerException("Failed to cancel tokens " + e.getMessage() + e.getCause(), e); } }
From source file:azkaban.security.HadoopSecurityManager_H_2_0.java
License:Apache License
@Override public synchronized void prefetchToken(final File tokenFile, final Props props, final Logger logger) throws HadoopSecurityManagerException { final String userToProxy = props.getString(USER_TO_PROXY); logger.info("Getting hadoop tokens based on props for " + userToProxy); final Credentials cred = new Credentials(); if (props.getBoolean(OBTAIN_HCAT_TOKEN, false)) { try {//from ww w .j a v a2 s . c o m // first we fetch and save the default hcat token. logger.info("Pre-fetching default Hive MetaStore token from hive"); HiveConf hiveConf = new HiveConf(); Token<DelegationTokenIdentifier> hcatToken = fetchHcatToken(userToProxy, hiveConf, null, logger); cred.addToken(hcatToken.getService(), hcatToken); // check and see if user specified the extra hcat locations we need to // look at and fetch token. final List<String> extraHcatLocations = props.getStringList(EXTRA_HCAT_LOCATION); if (Collections.EMPTY_LIST != extraHcatLocations) { logger.info("Need to pre-fetch extra metaStore tokens from hive."); // start to process the user inputs. for (String thriftUrl : extraHcatLocations) { logger.info("Pre-fetching metaStore token from : " + thriftUrl); hiveConf = new HiveConf(); hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, thriftUrl); hcatToken = fetchHcatToken(userToProxy, hiveConf, thriftUrl, logger); cred.addToken(hcatToken.getService(), hcatToken); } } } catch (Throwable t) { String message = "Failed to get hive metastore token." + t.getMessage() + t.getCause(); logger.error(message, t); throw new HadoopSecurityManagerException(message); } } if (props.getBoolean(OBTAIN_JOBHISTORYSERVER_TOKEN, false)) { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); logger.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); logger.info("Pre-fetching JH token from job history server"); Token<?> jhsdt = null; try { jhsdt = getDelegationTokenFromHS(hsProxy); } catch (Exception e) { logger.error("Failed to fetch JH token", e); throw new HadoopSecurityManagerException("Failed to fetch JH token for " + userToProxy); } if (jhsdt == null) { logger.error("getDelegationTokenFromHS() returned null"); throw new HadoopSecurityManagerException("Unable to fetch JH token for " + userToProxy); } logger.info("Created JH token: " + jhsdt.toString()); logger.info("Token kind: " + jhsdt.getKind()); logger.info("Token id: " + jhsdt.getIdentifier()); logger.info("Token service: " + jhsdt.getService()); cred.addToken(jhsdt.getService(), jhsdt); } try { getProxiedUser(userToProxy).doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { getToken(userToProxy); return null; } private void getToken(String userToProxy) throws InterruptedException, IOException, HadoopSecurityManagerException { logger.info("Here is the props for " + OBTAIN_NAMENODE_TOKEN + ": " + props.getBoolean(OBTAIN_NAMENODE_TOKEN)); if (props.getBoolean(OBTAIN_NAMENODE_TOKEN, false)) { FileSystem fs = FileSystem.get(conf); // check if we get the correct FS, and most importantly, the // conf logger.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs .getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { logger.error("Failed to fetch DFS token for "); throw new HadoopSecurityManagerException( "Failed to fetch DFS token for " + userToProxy); } logger.info("Created DFS token: " + fsToken.toString()); logger.info("Token kind: " + fsToken.getKind()); logger.info("Token id: " + fsToken.getIdentifier()); logger.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); // getting additional name nodes tokens String otherNamenodes = props.get(OTHER_NAMENODES_TO_GET_TOKEN); if ((otherNamenodes != null) && (otherNamenodes.length() > 0)) { logger.info(OTHER_NAMENODES_TO_GET_TOKEN + ": '" + otherNamenodes + "'"); String[] nameNodeArr = otherNamenodes.split(","); Path[] ps = new Path[nameNodeArr.length]; for (int i = 0; i < ps.length; i++) { ps[i] = new Path(nameNodeArr[i].trim()); } TokenCache.obtainTokensForNamenodes(cred, ps, conf); logger.info("Successfully fetched tokens for: " + otherNamenodes); } else { logger.info(OTHER_NAMENODES_TO_GET_TOKEN + " was not configured"); } } if (props.getBoolean(OBTAIN_JOBTRACKER_TOKEN, false)) { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); logger.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient .getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { logger.error("Failed to fetch JT token"); throw new HadoopSecurityManagerException("Failed to fetch JT token for " + userToProxy); } logger.info("Created JT token: " + mrdt.toString()); logger.info("Token kind: " + mrdt.getKind()); logger.info("Token id: " + mrdt.getIdentifier()); logger.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } } }); FileOutputStream fos = null; DataOutputStream dos = null; try { fos = new FileOutputStream(tokenFile); dos = new DataOutputStream(fos); cred.writeTokenStorageToStream(dos); } finally { if (dos != null) { try { dos.close(); } catch (Throwable t) { // best effort logger.error("encountered exception while closing DataOutputStream of the tokenFile", t); } } if (fos != null) { fos.close(); } } // stash them to cancel after use. logger.info("Tokens loaded in " + tokenFile.getAbsolutePath()); } catch (Exception e) { throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + e.getMessage() + e.getCause(), e); } catch (Throwable t) { throw new HadoopSecurityManagerException( "Failed to get hadoop tokens! " + t.getMessage() + t.getCause(), t); } }
From source file:gobblin.hadoop.token.TokenUtils.java
License:Open Source License
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf);// www . j av a 2 s . c o m final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + jhToken.getIdentifier()); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
From source file:gobblin.hadoop.token.TokenUtils.java
License:Open Source License
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); }/*w ww . j av a 2 s . c o m*/ LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + fsToken.getIdentifier()); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
From source file:gobblin.hadoop.token.TokenUtils.java
License:Open Source License
private static void getJtToken(Credentials cred) throws IOException { try {/*from w ww . j a va 2 s .c om*/ JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); LOG.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient .getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { LOG.error("Failed to fetch JT token"); throw new IOException("Failed to fetch JT token."); } LOG.info("Created JT token: " + mrdt.toString()); LOG.info("Token kind: " + mrdt.getKind()); LOG.info("Token id: " + mrdt.getIdentifier()); LOG.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } catch (InterruptedException ie) { throw new IOException(ie); } }