Example usage for org.apache.hadoop.fs FileUtil chmod

List of usage examples for org.apache.hadoop.fs FileUtil chmod

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileUtil
  6. chmod

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil chmod.

Prototype

public static int chmod(String filename, String perm) throws IOException, InterruptedException

Source Link

Document

Change the permissions on a filename.

Usage

From source file:UnmanagedAMLauncher.java

License:Apache License

public void launchAM(ApplicationAttemptId attemptId) throws IOException, YarnException {
    Credentials credentials = new Credentials();
    Token<AMRMTokenIdentifier> token = rmClient.getAMRMToken(attemptId.getApplicationId());
    // Service will be empty but that's okay, we are just passing down only
    // AMRMToken down to the real AM which eventually sets the correct
    // service-address.
    credentials.addToken(token.getService(), token);
    File tokenFile = File.createTempFile("unmanagedAMRMToken", "", new File(System.getProperty("user.dir")));
    try {/*from w w  w  .ja  va  2s .c  o  m*/
        FileUtil.chmod(tokenFile.getAbsolutePath(), "600");
    } catch (InterruptedException ex) {
        throw new RuntimeException(ex);
    }
    tokenFile.deleteOnExit();
    DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true));
    credentials.writeTokenStorageToStream(os);
    os.close();

    Map<String, String> env = System.getenv();
    ArrayList<String> envAMList = new ArrayList<String>();
    boolean setClasspath = false;
    for (Map.Entry<String, String> entry : env.entrySet()) {
        String key = entry.getKey();
        String value = entry.getValue();
        if (key.equals("CLASSPATH")) {
            setClasspath = true;
            if (classpath != null) {
                value = value + File.pathSeparator + classpath;
            }
        }
        envAMList.add(key + "=" + value);
    }

    if (!setClasspath && classpath != null) {
        envAMList.add("CLASSPATH=" + classpath);
    }
    ContainerId containerId = ContainerId.newContainerId(attemptId, 0);

    String hostname = InetAddress.getLocalHost().getHostName();
    envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId);
    envAMList.add(Environment.NM_HOST.name() + "=" + hostname);
    envAMList.add(Environment.NM_HTTP_PORT.name() + "=0");
    envAMList.add(Environment.NM_PORT.name() + "=0");
    envAMList.add(Environment.LOCAL_DIRS.name() + "= /tmp");
    envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + System.currentTimeMillis());

    envAMList.add(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME + "=" + tokenFile.getAbsolutePath());

    String[] envAM = new String[envAMList.size()];
    Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM));

    final BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.getErrorStream()));
    final BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.getInputStream()));

    // read error and input streams as this would free up the buffers
    // free the error stream buffer
    Thread errThread = new Thread() {
        @Override
        public void run() {
            try {
                String line = errReader.readLine();
                while ((line != null) && !isInterrupted()) {
                    System.err.println(line);
                    line = errReader.readLine();
                }
            } catch (IOException ioe) {
                LOG.warn("Error reading the error stream", ioe);
            }
        }
    };
    Thread outThread = new Thread() {
        @Override
        public void run() {
            try {
                String line = inReader.readLine();
                while ((line != null) && !isInterrupted()) {
                    System.out.println(line);
                    line = inReader.readLine();
                }
            } catch (IOException ioe) {
                LOG.warn("Error reading the out stream", ioe);
            }
        }
    };
    try {
        errThread.start();
        outThread.start();
    } catch (IllegalStateException ise) {
    }

    // wait for the process to finish and check the exit code
    try {
        int exitCode = amProc.waitFor();
        LOG.info("AM process exited with value: " + exitCode);
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        amCompleted = true;
    }

    try {
        // make sure that the error thread exits
        // on Windows these threads sometimes get stuck and hang the execution
        // timeout and join later after destroying the process.
        errThread.join();
        outThread.join();
        errReader.close();
        inReader.close();
    } catch (InterruptedException ie) {
        LOG.info("ShellExecutor: Interrupted while reading the error/out stream", ie);
    } catch (IOException ioe) {
        LOG.warn("Error while closing the error/out stream", ioe);
    }
    amProc.destroy();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.//  w w  w.  j av a2 s  .c o  m
 * @param job
 *        contains BSPJob configuration
 * @param communicator
 *        the communicator between different work node
 */
public Application(BSPJob job, Communicator communicator) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    List<String> cmd = new ArrayList<String>();
    String executable = job.getJobExe();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler(communicator);
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor./*from  w  w  w.  j av  a  2  s. c o  m*/
 * @param job
 *        contains BSPJob configuration
 */
public Application(BSPJob job) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    List<String> cmd = new ArrayList<String>();
    String executable = job.getJobExe();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);

    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler();
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.//from   ww w .j  a v a  2 s .c o  m
 * @param job
 *        contains BSPJob configuration
 * @param processType
 *        the type of c++ process(for staff or workmanager)
 */
public Application(BSPJob job, String processType) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    env.put("processType", processType);
    String bcbspdir = job.getConf().get("bcbsp.log.dir");
    LOG.info("bcbsp log dir : " + bcbspdir);
    env.put("bcbsp.log.dir", bcbspdir);
    List<String> cmd = new ArrayList<String>();
    String executable = job.getJobExe();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);

    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler();
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.//from   ww w .j av a  2 s  .  c  o  m
 * @param job
 *        contains BSPJob configuration
 * @param processType
 *        the type of c++ process(for staff or workmanager)
 * @param jobCpath
 *        c++ executable file path
 */
public Application(BSPJob job, String processType, String jobCpath) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    env.put("processType", processType);
    String bcbspdir = job.getConf().get("bcbsp.log.dir");
    LOG.info("bcbsp log dir : " + bcbspdir);
    env.put("bcbsp.log.dir", bcbspdir);
    List<String> cmd = new ArrayList<String>();
    String executable = jobCpath;
    LOG.info("processType is :" + processType);
    LOG.info("executable is :" + executable);
    // String executable = job.getJobExe();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler();
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.// ww w  .  j av a 2  s. com
 * @param conf
 *        contains Job configuration
 * @param processType
 *        the type of c++ process(for staff or workmanager)
 */
public Application(Configuration conf, String processType) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    env.put("processType", processType);
    List<String> cmd = new ArrayList<String>();
    String executable = conf.get(Constants.USER_BC_BSP_JOB_EXE);
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler();
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.//from   ww  w .  jav a 2s .  c o m
 * @param job
 *        contains BSPJob configuration
 * @param staff
 *        the java compute process
 * @param workerAgent
 *        Protocol that staff child process uses to contact its parent process
 */
public Application(BSPJob job, Staff staff, WorkerAgentProtocol workerAgent)
        throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    LOG.info("Application: System.getProterty: " + System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    env.put("staffID", staff.getStaffID().toString());
    LOG.info("staffID is :" + staff.getStaffID().toString());
    List<String> cmd = new ArrayList<String>();
    String executable = staff.getJobExeLocalPath();
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    process = runClient(cmd, env);
    LOG.info("waiting for connect cpp process ");
    clientSocket = serverSocket.accept();
    LOG.info("=========run C++ ======");
    this.handler = new TaskHandler(job, staff, workerAgent);
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor.//from   w  w  w. j a va 2 s . co m
 * @param job
 *        contains BSPJob configuration
 * @param staff
 *        the java compute process
 * @param workerAgent
 *        Protocol that staff child process uses to contact its parent process
 * @param processType
 *        the type of c++ process(for staff or workmanager)
 */
public Application(BSPJob job, Staff staff, WorkerAgentProtocol workerAgent, String processType)
        throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    LOG.info("Application: System.getProterty: " + System.getProperty("java.io.tmpdir"));
    env.put("processType", processType);
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    String bcbspdir = job.getConf().get("bcbsp.log.dir");
    String userdefine = job.getConf().get("userDefine");
    env.put("userDefine", userdefine);
    LOG.info("bcbsp log dir : " + bcbspdir);
    env.put("bcbsp.log.dir", bcbspdir);
    if (processType.equalsIgnoreCase("staff")) {
        env.put("staffID", staff.getStaffID().toString());
        LOG.info("staffID is :" + staff.getStaffID().toString());
    } else {
    }
    List<String> cmd = new ArrayList<String>();
    String executable = staff.getJobExeLocalPath();
    File file = new File(executable);
    if (file.exists()) {
        LOG.info("the jobC is exist");
    } else {
        LOG.info("the jobC is not exist");
    }
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);

    process = runClient(cmd, env);
    LOG.info("waiting for connect cpp process");
    clientSocket = serverSocket.accept();
    LOG.info("=========run C++ ======");
    this.handler = new TaskHandler(job, staff, workerAgent);
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

From source file:it.crs4.pydoop.mapreduce.pipes.Application.java

License:Apache License

/**
 * Start the child process to handle the task for us.
 * @throws IOException//from  ww w  . j  a va2  s . co m
 * @throws InterruptedException
 */
Application(TaskInputOutputContext<K1, V1, K2, V2> context, DummyRecordReader input)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    // add TMPDIR environment variable with the value of java.io.tmpdir
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put(Submitter.PORT, Integer.toString(serverSocket.getLocalPort()));

    //Add token to the environment if security is enabled
    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(context.getCredentials());
    // This password is used as shared secret key between this application and
    // child pipes process
    byte[] password = jobToken.getPassword();
    String localPasswordFile = new File(".") + Path.SEPARATOR + "jobTokenPassword";
    writePasswordToLocalFile(localPasswordFile, password, conf);
    // FIXME why is this not Submitter.SECRET_LOCATION ?
    env.put("hadoop.pipes.shared.secret.location", localPasswordFile);

    List<String> cmd = new ArrayList<String>();
    String interpretor = conf.get(Submitter.INTERPRETOR);
    if (interpretor != null) {
        cmd.add(interpretor);
    }
    String executable = context.getLocalCacheFiles()[0].toString();
    if (!(new File(executable).canExecute())) {
        // LinuxTaskController sets +x permissions on all distcache files already.
        // In case of DefaultTaskController, set permissions here.
        FileUtil.chmod(executable, "u+x");
    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt. 
    TaskAttemptID taskid = context.getTaskAttemptID();

    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
    cmd = TaskLog.captureOutAndError(null, cmd, stdout, stderr, logLength, false);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();

    String challenge = getSecurityChallenge();
    String digestToSend = createDigest(password, challenge);
    String digestExpected = createDigest(password, digestToSend);

    handler = new OutputHandler<K2, V2>(context, input, digestExpected);
    K2 outputKey = (K2) ReflectionUtils.newInstance(context.getOutputKeyClass(), conf);
    V2 outputValue = (V2) ReflectionUtils.newInstance(context.getOutputValueClass(), conf);
    downlink = new BinaryProtocol<K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue, conf);

    downlink.authenticate(digestToSend, challenge);
    waitForAuthentication();
    LOG.debug("Authentication succeeded");
    downlink.start();
    downlink.setJobConf(conf);
}

From source file:it.crs4.pydoop.mapreduce.pipes.TestPipeApplication.java

License:Apache License

private File[] cleanTokenPasswordFile() throws Exception {
    File[] result = new File[2];
    result[0] = new File("./jobTokenPassword");
    if (result[0].exists()) {
        FileUtil.chmod(result[0].getAbsolutePath(), "700");
        assertTrue(result[0].delete());//w ww .  ja v a 2  s.c  o  m
    }
    result[1] = new File("./.jobTokenPassword.crc");
    if (result[1].exists()) {
        FileUtil.chmod(result[1].getAbsolutePath(), "700");
        result[1].delete();
    }
    return result;
}