Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:com.yss.yarn.launch.APPLaunch.java

License:Open Source License

private void launchApp(String appName, String queue, int amMB, String appMasterJar, String classPath,
        String lanchMainClass, Map<String, String> runenv) throws Exception {

    YarnClientApplication client_app = _yarn.createApplication();
    GetNewApplicationResponse app = client_app.getNewApplicationResponse();

    _appId = app.getApplicationId();//from  w  w w .j a  v  a2s  .co  m
    LOG.debug("_appId:" + _appId);

    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    appContext.setApplicationId(app.getApplicationId());
    appContext.setApplicationName(appName);
    appContext.setQueue(queue);

    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    LOG.info("load  " + appMasterJar);

    FileSystem fs = FileSystem.get(_hadoopConf);
    Path src = new Path(appMasterJar);
    String appHome = YarnUtil.getApplicationHomeForId(_appId.toString());
    Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar");

    fs.copyFromLocalFile(false, true, src, dst);
    localResources.put("AppMaster.jar", YarnUtil.newYarnAppResource(fs, dst));

    amContainer.setLocalResources(localResources);

    LOG.info("Set the environment for the application master");

    Map<String, String> env = new HashMap<String, String>();

    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar");

    List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath");

    LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]");

    String yarn_class_path = classPath;

    LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path);

    String java_home = System.getenv("JAVA_HOME");

    if ((java_home != null) && !java_home.isEmpty()) {
        env.put("JAVA_HOME", java_home);
    }

    LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]");
    env.put("appJar", appMasterJar);
    env.put("appName", appName);
    env.put("appId", _appId.toString());
    env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    env.putAll(runenv);
    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master
    Vector<String> vargs = new Vector<String>();

    if ((java_home != null) && !java_home.isEmpty()) {
        vargs.add(env.get("JAVA_HOME") + "/bin/java");
    } else {
        vargs.add("java");
    }

    vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log");

    vargs.add(lanchMainClass);
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Set java executable command
    LOG.info("Setting up app master command:" + vargs);
    amContainer.setCommands(vargs);

    Resource capability = Records.newRecord(Resource.class);

    appContext.setResource(capability);
    appContext.setAMContainerSpec(amContainer);
    appContext.setApplicationName(appName);
    _yarn.submitApplication(appContext);
}

---

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputCommitter.java

License:Apache License

/**
 * Create a file output committer/*from w w w  . j  a v  a 2s.com*/
 *
 * @param outputPath the job's output path
 * @param context the task's context
 * @throws IOException
 */
public HirodsFileOutputCommitter(Path outputPath, Path tempPath, TaskAttemptContext context)
        throws IOException {
    if (outputPath != null && tempPath != null) {
        this.outputPath = outputPath;
        this.outputFileSystem = outputPath.getFileSystem(context.getConfiguration());
        this.tempPath = tempPath;
        this.workFileSystem = tempPath.getFileSystem(context.getConfiguration());
        this.workPath = new Path(tempPath, (HirodsFileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_"
                + context.getTaskAttemptID().toString())).makeQualified(this.workFileSystem);
    }
}

---

From source file:edu.snu.dolphin.bsp.core.OutputStreamProviderHDFS.java

License:Apache License

@Override
public DataOutputStream create(final String name) throws IOException {
    final String directoryPath = outputPath + Path.SEPARATOR + name;
    if (!fs.exists(new Path(directoryPath))) {
        fs.mkdirs(new Path(directoryPath));
    }/*  www .  j ava 2  s  .  co m*/
    return fs.create(new Path(directoryPath + Path.SEPARATOR + taskId));
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

public boolean init() throws ParseException, IOException, AsterixException, YarnException {
    try {/*from  w  w w  .j ava2  s.  c o  m*/
        localizeDFSResources();
        clusterDesc = Utils.parseYarnClusterConfig(CLUSTER_DESC_PATH);
        cC = clusterDesc.getMasterNode();
        appMasterTrackingUrl = "http://" + cC.getClientIp() + ":" + cC.getClientPort() + Path.SEPARATOR;
        distributeAsterixConfig();
        //now let's read what's in there so we can set the JVM opts right
        LOG.debug("config file loc: " + System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY));
    } catch (FileNotFoundException | IllegalStateException e) {
        LOG.error("Could not deserialize Cluster Config from disk- aborting!");
        LOG.error(e);
        throw e;
    }

    return true;
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

/**
 * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client
 * filesystem of the nodes.// ww  w.  j  a v  a 2 s  .c o m
 * 
 * @throws IOException
 */
private void localizeDFSResources() throws IOException {
    //if performing an 'offline' task, skip a lot of resource distribution
    if (obliterate || backup || restore) {
        if (appMasterJar == null || ("").equals(appMasterJar)) {
            //this can happen in a jUnit testing environment. we don't need to set it there. 
            if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
                throw new IllegalStateException("AM jar not provided in environment.");
            } else {
                return;
            }
        }
        FileSystem fs = FileSystem.get(conf);
        FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar);
        LocalResource obliteratorJar = Records.newRecord(LocalResource.class);
        obliteratorJar.setType(LocalResourceType.FILE);
        obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE);
        obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar));
        obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime());
        obliteratorJar.setSize(appMasterJarStatus.getLen());
        localResources.put("asterix-yarn.jar", obliteratorJar);
        LOG.info(localResources.values());
        return;
    }
    //otherwise, distribute evertything to start up asterix

    LocalResource asterixZip = Records.newRecord(LocalResource.class);

    //this un-tar's the asterix distribution
    asterixZip.setType(LocalResourceType.ARCHIVE);

    asterixZip.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath)));

    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath);
        throw new IOException(e);
    }

    asterixZip.setTimestamp(asterixZipTimestamp);
    asterixZip.setSize(asterixZipLen);
    localResources.put(ASTERIX_ZIP_NAME, asterixZip);

    //now let's do the same for the cluster description XML
    LocalResource asterixConf = Records.newRecord(LocalResource.class);
    asterixConf.setType(LocalResourceType.FILE);

    asterixConf.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath)));
    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath);
        throw new IOException(e);
    }
    //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node. 
    asterixConf.setTimestamp(asterixConfTimestamp);
    asterixConf.setSize(asterixConfLen);
    localResources.put("cluster-config.xml", asterixConf);
    //now add the libraries if there are any
    try {
        FileSystem fs = FileSystem.get(conf);
        Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR);
        if (fs.exists(p)) {
            FileStatus[] dataverses = fs.listStatus(p);
            for (FileStatus d : dataverses) {
                if (!d.isDirectory())
                    throw new IOException("Library configuration directory structure is incorrect");
                FileStatus[] libraries = fs.listStatus(d.getPath());
                for (FileStatus l : libraries) {
                    if (l.isDirectory())
                        throw new IOException("Library configuration directory structure is incorrect");
                    LocalResource lr = Records.newRecord(LocalResource.class);
                    lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri()));
                    lr.setSize(l.getLen());
                    lr.setTimestamp(l.getModificationTime());
                    lr.setType(LocalResourceType.ARCHIVE);
                    lr.setVisibility(LocalResourceVisibility.PRIVATE);
                    localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR
                            + l.getPath().getName().split("\\.")[0], lr);
                    LOG.info("Found library: " + l.getPath().toString());
                    LOG.info(l.getPath().getName());
                }
            }
        }
    } catch (FileNotFoundException e) {
        LOG.info("No external libraries present");
        //do nothing, it just means there aren't libraries. that is possible and ok
        // it should be handled by the fs.exists(p) check though.
    }
    LOG.info(localResources.values());

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Upload External libraries and functions to HDFS for an instance to use when started
 * @throws IllegalStateException// w  ww .j a v a2s.co m
 * @throws IOException
 */

private void installExtLibs() throws IllegalStateException, IOException {
    FileSystem fs = FileSystem.get(conf);
    if (!instanceExists()) {
        throw new IllegalStateException("No instance by name " + instanceName + " found.");
    }
    if (isRunning()) {
        throw new IllegalStateException(
                "Instance " + instanceName + " is running. Please stop it before installing any libraries.");
    }
    String libPathSuffix = CONF_DIR_REL + instanceFolder + "library" + Path.SEPARATOR + libDataverse
            + Path.SEPARATOR;
    Path src = new Path(extLibs);
    String fullLibPath = libPathSuffix + src.getName();
    Path libFilePath = new Path(fs.getHomeDirectory(), fullLibPath);
    LOG.info("Copying Asterix external library to DFS");
    fs.copyFromLocalFile(false, true, src, libFilePath);
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Finds the minimal classes and JARs needed to start the AM only.
 * @return Resources the AM needs to start on the initial container.
 * @throws IllegalStateException//from  www .j  av  a  2s .c o  m
 * @throws IOException
 */
private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException {
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    FileSystem fs = FileSystem.get(conf);
    String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR;
    String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator"));
    String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar
    String commonsJarPattern = "^(commons).*(jar)$";
    String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests
    String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$";

    LOG.info(File.separator);
    for (String j : cp) {
        String[] pathComponents = j.split(Pattern.quote(File.separator));
        LOG.info(j);
        LOG.info(pathComponents[pathComponents.length - 1]);
        if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern)
                || pathComponents[pathComponents.length - 1].matches(commonsJarPattern)
                || pathComponents[pathComponents.length - 1].matches(surefireJarPattern)
                || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) {
            LOG.info("Loading JAR/classpath: " + j);
            File f = new File(j);
            Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName());
            if (!fs.exists(dst) || refresh) {
                fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst);
            }
            FileStatus dstSt = fs.getFileStatus(dst);
            LocalResource amLib = Records.newRecord(LocalResource.class);
            amLib.setType(LocalResourceType.FILE);
            amLib.setVisibility(LocalResourceVisibility.PRIVATE);
            amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst));
            amLib.setTimestamp(dstSt.getModificationTime());
            amLib.setSize(dstSt.getLen());
            DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate();
            amLibCoord.res = amLib;
            amLibCoord.name = f.getName();
            if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) {
                amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION);
                amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN);
                amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()),
                        AConstants.APPLICATIONMASTERJARTIMESTAMP);
            }
            resources.add(amLibCoord);
        }

    }
    if (resources.size() == 0) {
        throw new IOException("Required JARs are missing. Please check your directory structure");
    }
    return resources;
}

---

From source file:es.tid.cosmos.platform.injection.server.HadoopFileSystemView.java

License:Open Source License

private HadoopSshFile getFile(String baseDir, String file) throws IOException, InterruptedException {
    String requestedDir = baseDir;
    String requestedFile = file;/*w  ww  .  ja va 2  s . com*/

    if (requestedDir.isEmpty() && (requestedFile.isEmpty() || requestedFile.equals(Path.CUR_DIR))) {
        requestedDir = this.homePath;
        requestedFile = "";
        LOG.debug("redirecting to home path: " + this.homePath);
    } // if

    String wholePath = requestedDir + requestedFile;

    if (!requestedDir.endsWith(Path.SEPARATOR) && !requestedFile.startsWith(Path.SEPARATOR)) {
        wholePath = requestedDir + Path.SEPARATOR + requestedFile;
    } // if

    return new HadoopSshFile(wholePath, this.userName, this.hadoopFS);
}

---

From source file:es.tid.cosmos.platform.injection.server.HadoopSshFile.java

License:Open Source License

/**
 * /* ww w .  j  ava2 s .  co m*/
 * @param fileName
 * @return
 */
private String appendToPath(String fileName) {
    String absPath = this.getAbsolutePath();
    return ((absPath.endsWith(Path.SEPARATOR)) ? absPath : (absPath + Path.SEPARATOR)) + fileName;
}

---

From source file:eu.stratosphere.hadoopcompatibility.FileOutputCommitterWrapper.java

License:Apache License

public Path getTempTaskOutputPath(JobConf conf, TaskAttemptID taskAttemptID) {
    Path outputPath = FileOutputFormat.getOutputPath(conf);
    if (outputPath != null) {
        Path p = new Path(outputPath,
                (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskAttemptID.toString()));
        try {/*ww w  .j  a va2s . co m*/
            FileSystem fs = p.getFileSystem(conf);
            return p.makeQualified(fs);
        } catch (IOException ie) {
            LOG.warn(StringUtils.stringifyException(ie));
            return p;
        }
    }
    return null;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »