Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:com.uber.hoodie.common.minicluster.HdfsTestService.java

License:Apache License

/**
 * Get the location on the local FS where we store the HDFS data.
 *
 * @param baseFsLocation The base location on the local filesystem we have write access to create dirs.
 * @return The location for HDFS data.//from   w ww.  java2  s. c o  m
 */
private static String getDFSLocation(String baseFsLocation) {
    return baseFsLocation + Path.SEPARATOR + "dfs";
}

---

From source file:com.uber.hoodie.hive.util.HiveTestService.java

License:Apache License

private static String getHiveLocation(String baseLocation) {
    return baseLocation + Path.SEPARATOR + "hive";
}

---

From source file:com.xiaomi.linden.hadoop.indexing.job.LindenJob.java

License:Apache License

public static Shard[] createShards(String indexPath, int numShards) throws IOException {
    String indexSubDirPrefix = "shard";
    String parent = Shard.normalizePath(indexPath) + Path.SEPARATOR;
    Shard[] shards = new Shard[numShards];
    for (int i = 0; i < shards.length; i++) {
        shards[i] = new Shard(parent + indexSubDirPrefix + NUMBER_FORMAT.format(i));
    }/*from   w ww .  ja va 2 s. c o m*/
    return shards;
}

---

From source file:com.xiaomi.linden.hadoop.indexing.reduce.LindenReducer.java

License:Apache License

@Override
protected void reduce(Shard key, Iterable<IntermediateForm> values, Context context)
        throws IOException, InterruptedException {

    logger.info("Construct a shard writer for " + key);
    FileSystem fs = FileSystem.get(conf);
    // debug://from   www  .j av  a 2 s .c om
    logger.info("filesystem is: " + fs.getUri());
    String temp = mapreduceuceTempDir + Path.SEPARATOR + "shard_" + key.toFlatString() + "_"
            + System.currentTimeMillis();
    logger.info("mapreduceuceTempDir is: " + mapreduceuceTempDir);
    final ShardWriter writer = new ShardWriter(fs, key, temp, conf);

    // update the shard
    Iterator<IntermediateForm> iterator = values.iterator();
    while (iterator.hasNext()) {
        IntermediateForm form = iterator.next();
        writer.process(form, facetsConfig);
    }

    // close the shard
    new Closeable() {
        volatile boolean closed = false;

        @Override
        public void close() throws IOException {
            // spawn a thread to give progress heartbeats
            Thread prog = new Thread() {
                @Override
                public void run() {
                    while (!closed) {
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            continue;
                        } catch (Throwable e) {
                            return;
                        }
                    }
                }
            };

            try {
                prog.start();
                if (writer != null) {
                    writer.optimize(); // added this option to optimize after all the docs have been added;
                    writer.close();
                }
            } finally {
                closed = true;
            }
        }
    }.close();
    logger.info("Closed the shard writer for " + key + ", writer = " + writer);
    context.write(key, DONE);
}

---

From source file:com.yahoo.storm.yarn.StormOnYarn.java

License:Open Source License

private void launchApp(String appName, String queue, int amMB, String storm_zip_location) throws Exception {
    LOG.debug("StormOnYarn:launchApp() ...");
    YarnClientApplication client_app = _yarn.createApplication();
    GetNewApplicationResponse app = client_app.getNewApplicationResponse();
    _appId = app.getApplicationId();// w w  w.ja  v  a2 s  . c o  m
    LOG.debug("_appId:" + _appId);

    if (amMB > app.getMaximumResourceCapability().getMemory()) {
        //TODO need some sanity checks
        amMB = app.getMaximumResourceCapability().getMemory();
    }
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
    appContext.setApplicationId(app.getApplicationId());
    appContext.setApplicationName(appName);
    appContext.setQueue(queue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the
    // local resources
    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    String appMasterJar = findContainingJar(MasterServer.class);
    FileSystem fs = FileSystem.get(_hadoopConf);
    Path src = new Path(appMasterJar);
    String appHome = Util.getApplicationHomeForId(_appId.toString());
    Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar");
    fs.copyFromLocalFile(false, true, src, dst);
    localResources.put("AppMaster.jar", Util.newYarnAppResource(fs, dst));

    String stormVersion = Util.getStormVersion();
    Path zip;
    if (storm_zip_location != null) {
        zip = new Path(storm_zip_location);
    } else {
        zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    }
    _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath());
    LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC;
    _stormConf.put("storm.zip.visibility", "PUBLIC");
    if (!Util.isPublic(fs, zip)) {
        visibility = LocalResourceVisibility.APPLICATION;
        _stormConf.put("storm.zip.visibility", "APPLICATION");
    }
    localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, visibility));

    Path confDst = Util.createConfigurationFileInFs(fs, appHome, _stormConf, _hadoopConf);
    // establish a symbolic link to conf directory
    localResources.put("conf", Util.newYarnAppResource(fs, confDst));

    // Setup security tokens
    Path[] paths = new Path[3];
    paths[0] = dst;
    paths[1] = zip;
    paths[2] = confDst;
    Credentials credentials = new Credentials();
    TokenCache.obtainTokensForNamenodes(credentials, paths, _hadoopConf);
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    //security tokens for HDFS distributed cache
    amContainer.setTokens(securityTokens);

    // Set local resource info into app master container launch context
    amContainer.setLocalResources(localResources);

    // Set the env variables to be setup in the env where the application master
    // will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    // add the runtime classpath needed for tests to work
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./conf");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar");

    //Make sure that AppMaster has access to all YARN JARs
    List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath");
    ProcessBuilder pb = new ProcessBuilder(yarn_classpath_cmd).redirectError(Redirect.INHERIT);
    LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]");
    pb.environment().putAll(System.getenv());
    Process proc = pb.start();
    BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), "UTF-8"));
    String line = "";
    String yarn_class_path = (String) _stormConf.get("storm.yarn.yarn_classpath");
    if (yarn_class_path == null) {
        StringBuilder yarn_class_path_builder = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            yarn_class_path_builder.append(line);
        }
        yarn_class_path = yarn_class_path_builder.toString();
    }
    LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]");
    proc.waitFor();
    reader.close();
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path);

    String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion);
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/*");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/lib/*");

    String java_home = (String) _stormConf.get("storm.yarn.java_home");
    if (java_home == null)
        java_home = System.getenv("JAVA_HOME");

    if (java_home != null && !java_home.isEmpty())
        env.put("JAVA_HOME", java_home);
    LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]");

    env.put("appJar", appMasterJar);
    env.put("appName", appName);
    env.put("appId", new Integer(_appId.getId()).toString());
    env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master
    Vector<String> vargs = new Vector<String>();
    if (java_home != null && !java_home.isEmpty())
        vargs.add(env.get("JAVA_HOME") + "/bin/java");
    else
        vargs.add("java");
    vargs.add("-Dstorm.home=./storm/" + stormHomeInZip + "/");
    vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log");
    //vargs.add("-verbose:class");
    vargs.add("com.yahoo.storm.yarn.MasterServer");
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    // Set java executable command
    LOG.info("Setting up app master command:" + vargs);

    amContainer.setCommands(vargs);

    // Set up resource type requirements
    // For now, only memory is supported so we set memory requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMB);
    appContext.setResource(capability);
    appContext.setAMContainerSpec(amContainer);

    _yarn.submitApplication(appContext);
}

---

From source file:com.yahoo.storm.yarn.TestConfig.java

License:Open Source License

private void unzipFile(String filePath) {
    FileInputStream fis = null;/* ww w .j a v  a  2  s  .com*/
    ZipInputStream zipIs = null;
    ZipEntry zEntry = null;
    try {
        fis = new FileInputStream(filePath);
        zipIs = new ZipInputStream(new BufferedInputStream(fis));
        while ((zEntry = zipIs.getNextEntry()) != null) {
            try {
                byte[] tmp = new byte[4 * 1024];
                FileOutputStream fos = null;
                String opFilePath = "lib/" + zEntry.getName();
                if (zEntry.isDirectory()) {
                    LOG.debug("Create a folder " + opFilePath);
                    if (zEntry.getName().indexOf(Path.SEPARATOR) == (zEntry.getName().length() - 1))
                        storm_home = opFilePath.substring(0, opFilePath.length() - 1);
                    new File(opFilePath).mkdir();
                } else {
                    LOG.debug("Extracting file to " + opFilePath);
                    fos = new FileOutputStream(opFilePath);
                    int size = 0;
                    while ((size = zipIs.read(tmp)) != -1) {
                        fos.write(tmp, 0, size);
                    }
                    fos.flush();
                    fos.close();
                }
            } catch (Exception ex) {

            }
        }
        zipIs.close();
    } catch (FileNotFoundException e) {
        LOG.warn(e.toString());
    } catch (IOException e) {
        LOG.warn(e.toString());
    }
    LOG.info("storm_home: " + storm_home);
}

---

From source file:com.yahoo.storm.yarn.Util.java

License:Open Source License

@SuppressWarnings("rawtypes")
static Path createConfigurationFileInFs(FileSystem fs, String appHome, Map stormConf,
        YarnConfiguration yarnConf) throws IOException {
    // dump stringwriter's content into FS conf/storm.yaml
    Path confDst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING);
    Path dirDst = confDst.getParent();
    fs.mkdirs(dirDst);/*from  w  ww. j  a v  a 2s .co  m*/

    //storm.yaml
    FSDataOutputStream out = fs.create(confDst);
    Yaml yaml = new Yaml();
    OutputStreamWriter writer = new OutputStreamWriter(out);
    rmNulls(stormConf);
    yaml.dump(stormConf, writer);
    writer.close();
    out.close();

    //yarn-site.xml
    Path yarn_site_xml = new Path(dirDst, "yarn-site.xml");
    out = fs.create(yarn_site_xml);
    writer = new OutputStreamWriter(out);
    yarnConf.writeXml(writer);
    writer.close();
    out.close();

    //logback.xml
    Path logback_xml = new Path(dirDst, "logback.xml");
    out = fs.create(logback_xml);
    CreateLogbackXML(out);
    out.close();

    return dirDst;
}

---

From source file:com.yahoo.storm.yarn.Util.java

License:Open Source License

static String getApplicationHomeForId(String id) {
    if (id.isEmpty()) {
        throw new IllegalArgumentException("The ID of the application cannot be empty.");
    }/*from  w w w.j  a va2s  . co m*/
    return ".storm" + Path.SEPARATOR + id;
}

---

From source file:com.yss.util.YarnUtil.java

License:Open Source License

@SuppressWarnings("rawtypes")
public static Path createConfigurationFileInFs(FileSystem fs, String appHome, Map stormConf,
        YarnConfiguration yarnConf) throws IOException {
    // dump stringwriter's content into FS conf/storm.yaml
    Path confDst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING);
    Path dirDst = confDst.getParent();
    fs.mkdirs(dirDst);/*from w  w  w.j  a v a  2s .co  m*/

    //storm.yaml
    FSDataOutputStream out = fs.create(confDst);
    Yaml yaml = new Yaml();
    OutputStreamWriter writer = new OutputStreamWriter(out);
    rmNulls(stormConf);

    yaml.dump(stormConf, writer);

    writer.close();
    out.close();

    //yarn-site.xml
    Path yarn_site_xml = new Path(dirDst, "yarn-site.xml");
    out = fs.create(yarn_site_xml);
    writer = new OutputStreamWriter(out);
    yarnConf.writeXml(writer);
    writer.close();
    out.close();
    return dirDst;
}

---

From source file:com.yss.util.YarnUtil.java

License:Open Source License

public static String getApplicationHomeForId(String id) {
    if (id.isEmpty()) {
        throw new IllegalArgumentException("The ID of the application cannot be empty.");
    }/*from  ww w .  j a va  2s. c  o m*/
    return ".storm" + Path.SEPARATOR + id;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »