List of usage examples for org.apache.hadoop.io.nativeio NativeIO isAvailable
public static boolean isAvailable()
From source file:com.asakusafw.bulkloader.common.BulkLoaderInitializer.java
License:Apache License
/** * Hadoop Cluster????/*from www .java 2 s .c o m*/ * ??????? * HadoopCluster???? * @param jobflowId ID * @param executionId ID * @param properties ??(?$ASAKUSA_HOME/bulkloader/conf/?????) * @return ?? */ public static boolean initHadoopCluster(String jobflowId, String executionId, List<String> properties) { // Eagerly initializes Hadoop native libraries NativeIO.isAvailable(); return initialize(jobflowId, executionId, properties, false, true, false, null); }
From source file:hdfs.FileUtil.java
License:Apache License
/** * Set permissions to the required value. Uses the java primitives instead * of forking if group == other.//from w ww . j a v a2 s. c om * @param f the file to change * @param permission the new permissions * @throws IOException */ public static void setPermission(File f, FsPermission permission) throws IOException { FsAction user = permission.getUserAction(); FsAction group = permission.getGroupAction(); FsAction other = permission.getOtherAction(); // use the native/fork if the group/other permissions are different // or if the native is available if (group != other || NativeIO.isAvailable()) { execSetPermission(f, permission); return; } boolean rv = true; // read perms rv = f.setReadable(group.implies(FsAction.READ), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { f.setReadable(user.implies(FsAction.READ), true); checkReturnValue(rv, f, permission); } // write perms rv = f.setWritable(group.implies(FsAction.WRITE), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { f.setWritable(user.implies(FsAction.WRITE), true); checkReturnValue(rv, f, permission); } // exec perms rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { f.setExecutable(user.implies(FsAction.EXECUTE), true); checkReturnValue(rv, f, permission); } }
From source file:hdfs.FileUtil.java
License:Apache License
private static void execSetPermission(File f, FsPermission permission) throws IOException { if (NativeIO.isAvailable()) { // NativeIO.chmod(f.getCanonicalPath(), permission.toShort()); } else {//from ww w .ja v a2 s . co m execCommand(f, Shell.SET_PERMISSION_COMMAND, String.format("%04o", permission.toShort())); } }
From source file:org.apache.hadoop.fs.FileUtil.java
License:Apache License
/** * Set permissions to the required value. Uses the java primitives instead * of forking if group == other.//from w w w. j a v a2s. co m * @param f the file to change * @param permission the new permissions * @throws java.io.IOException */ public static void setPermission(File f, FsPermission permission) throws IOException { FsAction user = permission.getUserAction(); FsAction group = permission.getGroupAction(); FsAction other = permission.getOtherAction(); // use the native/fork if the group/other permissions are different // or if the native is available if (group != other || NativeIO.isAvailable()) { execSetPermission(f, permission); return; } boolean rv = true; // read perms rv = f.setReadable(group.implies(FsAction.READ), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { f.setReadable(user.implies(FsAction.READ), true); checkReturnValue(rv, f, permission); } // write perms rv = f.setWritable(group.implies(FsAction.WRITE), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { f.setWritable(user.implies(FsAction.WRITE), true); checkReturnValue(rv, f, permission); } // exec perms rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); checkReturnValue(rv, f, permission); if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { f.setExecutable(user.implies(FsAction.EXECUTE), true); checkReturnValue(rv, f, permission); } }
From source file:org.apache.hadoop.fs.FileUtil.java
License:Apache License
private static void execSetPermission(File f, FsPermission permission) throws IOException { if (NativeIO.isAvailable()) { NativeIO.chmod(f.getCanonicalPath(), permission.toShort()); } else {/* w ww . j a v a 2 s . co m*/ execCommand(f, Shell.SET_PERMISSION_COMMAND, String.format("%04o", permission.toShort())); } }
From source file:org.apache.slider.common.tools.SliderUtils.java
License:Apache License
/** * Check for any needed libraries being present. On Unix none are needed; * on windows they must be present//from w w w.ja va 2 s. co m * @return true if all is well */ public static String checkForRequiredNativeLibraries() { if (!Shell.WINDOWS) { return ""; } StringBuilder errorText = new StringBuilder(""); if (!NativeIO.isAvailable()) { errorText.append("No native IO library. "); } try { String path = Shell.getQualifiedBinPath("winutils.exe"); log.debug("winutils is at {}", path); } catch (IOException e) { errorText.append("No WINUTILS.EXE. "); log.warn("No winutils: {}", e, e); } try { File target = new File("target"); FileUtil.canRead(target); } catch (UnsatisfiedLinkError e) { log.warn("Failing to link to native IO methods: {}", e, e); errorText.append("No native IO methods"); } return errorText.toString(); }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
/** * Validate the ownership of the map-output files being pulled in. The * local-file-system owner of the file should match the user component in the * * @throws Exception exception/* w ww. ja v a 2 s .c o m*/ */ @Test(timeout = 100000) public void testMapFileAccess() throws IOException { // This will run only in NativeIO is enabled as SecureIOUtils need it assumeTrue(NativeIO.isAvailable()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345, 1); LOG.info(appId.toString()); String appAttemptId = "attempt_12345_1_m_1_0"; String user = "randomUser"; String reducerId = "0"; List<File> fileMap = new ArrayList<File>(); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } }; } }; shuffleHandler.init(conf); try { shuffleHandler.start(); DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); byte[] byteArr = new byte[10000]; try { DataInputStream is = new DataInputStream(conn.getInputStream()); is.readFully(byteArr); } catch (EOFException e) { // ignore } // Retrieve file owner name FileInputStream is = new FileInputStream(fileMap.get(0)); String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner(); is.close(); String message = "Owner '" + owner + "' for path " + fileMap.get(0).getAbsolutePath() + " did not match expected owner '" + user + "'"; Assert.assertTrue((new String(byteArr)).contains(message)); } finally { shuffleHandler.stop(); FileUtil.fullyDelete(absLogDir); } }