List of usage examples for org.apache.hadoop.security UserGroupInformation createRemoteUser
@InterfaceAudience.Public @InterfaceStability.Evolving public static UserGroupInformation createRemoteUser(String user)
From source file:com.inforefiner.hdata.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException/*from ww w . j a va 2s. c o m*/ * @throws IOException */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException, InterruptedException { LOG.info("Starting ApplicationMaster"); // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class // are marked as LimitedPrivate Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); startTimelineClient(conf); if (timelineClient != null) { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); } // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainers); }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
@Test public void testPreserve() { try {/*from w w w . ja va 2 s.com*/ deleteState(); createSourceData(); UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest"); final CopyMapper copyMapper = new CopyMapper(); final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() { @Override public Mapper<Text, FileStatus, NullWritable, Text>.Context run() { try { StatusReporter reporter = new StubStatusReporter(); InMemoryWriter writer = new InMemoryWriter(); return getMapperContext(copyMapper, reporter, writer); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new RuntimeException(e); } } }); EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class); context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); touchFile(SOURCE_PATH + "/src/file.gz"); mkdirs(TARGET_PATH); cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511)); final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() { @Override public FileSystem run() { try { return FileSystem.get(configuration); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); throw new RuntimeException("Test ought to fail here"); } } }); tmpUser.doAs(new PrivilegedAction<Integer>() { @Override public Integer run() { try { copyMapper.setup(context); copyMapper.map(new Text("/src/file.gz"), tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context); Assert.fail("Expected copy to fail"); } catch (AccessControlException e) { Assert.assertTrue("Got exception: " + e.getMessage(), true); } catch (Exception e) { throw new RuntimeException(e); } return null; } }); } catch (Exception e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); } }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
@Test public void testCopyReadableFiles() { try {//from ww w . ja v a 2s. c o m deleteState(); createSourceData(); UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest"); final CopyMapper copyMapper = new CopyMapper(); final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() { @Override public Mapper<Text, FileStatus, NullWritable, Text>.Context run() { try { StatusReporter reporter = new StubStatusReporter(); InMemoryWriter writer = new InMemoryWriter(); return getMapperContext(copyMapper, reporter, writer); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new RuntimeException(e); } } }); touchFile(SOURCE_PATH + "/src/file.gz"); mkdirs(TARGET_PATH); cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file.gz"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ)); cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511)); final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() { @Override public FileSystem run() { try { return FileSystem.get(configuration); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); throw new RuntimeException("Test ought to fail here"); } } }); tmpUser.doAs(new PrivilegedAction<Integer>() { @Override public Integer run() { try { copyMapper.setup(context); copyMapper.map(new Text("/src/file.gz"), tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context); } catch (Exception e) { throw new RuntimeException(e); } return null; } }); } catch (Exception e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); } }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
@Test public void testSkipCopyNoPerms() { try {/*from w ww . j a v a2 s . c o m*/ deleteState(); createSourceData(); final InMemoryWriter writer = new InMemoryWriter(); UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest"); final CopyMapper copyMapper = new CopyMapper(); final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() { @Override public Mapper<Text, FileStatus, NullWritable, Text>.Context run() { try { StatusReporter reporter = new StubStatusReporter(); return getMapperContext(copyMapper, reporter, writer); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new RuntimeException(e); } } }); EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class); context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); touchFile(SOURCE_PATH + "/src/file.gz"); touchFile(TARGET_PATH + "/src/file.gz"); cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file.gz"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ)); cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file.gz"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ)); final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() { @Override public FileSystem run() { try { return FileSystem.get(configuration); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); throw new RuntimeException("Test ought to fail here"); } } }); tmpUser.doAs(new PrivilegedAction<Integer>() { @Override public Integer run() { try { copyMapper.setup(context); copyMapper.map(new Text("/src/file.gz"), tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context); } catch (Exception e) { throw new RuntimeException(e); } return null; } }); } catch (Exception e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); } }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
@Test public void testFailCopyWithAccessControlException() { try {/* ww w.j a v a 2s.c o m*/ deleteState(); createSourceData(); final InMemoryWriter writer = new InMemoryWriter(); UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest"); final CopyMapper copyMapper = new CopyMapper(); final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() { @Override public Mapper<Text, FileStatus, NullWritable, Text>.Context run() { try { StatusReporter reporter = new StubStatusReporter(); return getMapperContext(copyMapper, reporter, writer); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new RuntimeException(e); } } }); EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class); context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); touchFile(SOURCE_PATH + "/src/file"); OutputStream out = cluster.getFileSystem().create(new Path(TARGET_PATH + "/src/file")); out.write("hello world".getBytes()); out.close(); cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ)); cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file"), new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ)); final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() { @Override public FileSystem run() { try { return FileSystem.get(configuration); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); throw new RuntimeException("Test ought to fail here"); } } }); tmpUser.doAs(new PrivilegedAction<Integer>() { @Override public Integer run() { try { copyMapper.setup(context); copyMapper.map(new Text("/src/file"), tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context); Assert.fail("Didn't expect the file to be copied"); } catch (AccessControlException ignore) { } catch (Exception e) { if (e.getCause() == null || !(e.getCause() instanceof AccessControlException)) { throw new RuntimeException(e); } } return null; } }); } catch (Exception e) { LOG.error("Exception encountered ", e); Assert.fail("Test failed: " + e.getMessage()); } }
From source file:com.intel.databackend.datasources.hbase.HbaseConnManger.java
License:Apache License
private User getNoKrbUserFromSubject(Configuration configuration, String krbUser) throws IOException { return UserProvider.instantiate(configuration).create(UserGroupInformation.createRemoteUser(krbUser)); }
From source file:com.intel.hadoopRPCBenchmark.BenchmarkEngineTokenIdentifier.java
License:Apache License
@Override public UserGroupInformation getUser() { if (realUser.toString().isEmpty()) { return UserGroupInformation.createRemoteUser(tokenid.toString()); } else {// ww w.j a v a 2 s. c o m UserGroupInformation realUgi = UserGroupInformation.createRemoteUser(realUser.toString()); return UserGroupInformation.createProxyUser(tokenid.toString(), realUgi); } }
From source file:com.scistor.dshell.ScistorApplicationMaster.java
License:Apache License
/** * Main run function for the application master * //from ww w .j a va 2s . co m * @throws YarnException * @throws IOException */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Setup local RPC Server to accept status requests directly from // clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info("Received " + previousAMRunningContainers.size() + " previous AM's running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } }
From source file:com.sogou.dockeronyarn.service.DockerApplicationMaster_24.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException/*from ww w . ja v a 2s. c o m*/ * @throws IOException */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class // are marked as LimitedPrivate Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainers); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } numRetryCount.set(0); }
From source file:com.soteradefense.dga.DGAYarnRunner.java
License:Apache License
public static void main(final String[] args) throws Exception { if (!UserGroupInformation.isSecurityEnabled()) { UserGroupInformation.createRemoteUser(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER) .doAs(new PrivilegedAction<Void>() { @Override//from w w w.ja v a2s. c o m public Void run() { try { DGARunner runner = new DGARunner(); runner.run(args); } catch (Exception e) { e.printStackTrace(); } return null; } }); } else { DGARunner runner = new DGARunner(); runner.run(args); } }