Example usage for weka.core Environment substitute

Introduction

In this page you can find the example usage for weka.core Environment substitute.

Prototype

public String substitute(String source) throws Exception 

Source Link

Document

Substitute a variable names for their values in the given string.

Usage

From source file:distributed.hadoop.HDFSConfig.java

License:Open Source License

/**
 * Build an HDFS URL from the settings in the supplied HDFSConfig
 * //from  ww w .ja  v  a  2s. c o  m
 * @param config the HDFS config to use for building the URL
 * @param env environment variables
 * @return a HDFS URL as a String
 */
public static String constructHostURL(HDFSConfig config, Environment env) {
    String url = "hdfs://" + config.getHDFSHost() + ":" + config.getHDFSPort();

    try {
        if (env != null) {
            url = env.substitute(url);
        }
    } catch (Exception ex) {
    }

    return url;
}

---

From source file:distributed.hadoop.HDFSUtils.java

License:Open Source License

/**
 * Utility method to resolve all environment variables in a given path
 * // w  ww . ja  v  a  2s .  c o m
 * @param path the path in HDFS
 * @param env environment variables to use
 * @return the path with all environment variables resolved
 */
public static String resolvePath(String path, Environment env) {
    if (env != null) {
        try {
            path = env.substitute(path);
        } catch (Exception ex) {
        }
    }

    return path;

    // if (path.startsWith("hdfs://")) {
    // return path;
    // }
    //
    // String uri = "hdfs://" + config.getHDFSHost() + ":" +
    // config.getHDFSPort()
    // // + (path.startsWith("/") ? path : "/" + path);
    // + path;
    //
    // if (env != null) {
    // try {
    // uri = env.substitute(uri);
    // } catch (Exception ex) {
    // }
    // }

    // return uri;
}

---

From source file:distributed.hadoop.HDFSUtils.java

License:Open Source License

/**
 * Adds a file in HDFS to the classpath for hadoop nodes (via the
 * DistributedCache)//ww  w .  ja v  a 2 s  .  c  o  m
 * 
 * @param hdfsConfig the HDFSConfig object with host and port set
 * @param conf the Configuration object that will be changed by this operation
 * @param path the path to the file (in HDFS) to be added to the classpath for
 *          hadopp nodes
 * @param env any environment variables
 * @throws IOException if a problem occurs
 */
public static void addFileToClasspath(HDFSConfig hdfsConfig, Configuration conf, String path, Environment env)
        throws IOException {

    // conf.set(HDFSConfig.FS_DEFAULT_NAME,
    // HDFSConfig.constructHostURL(hdfsConfig, env));
    hdfsConfig.configureForHadoop(conf, env);

    FileSystem fs = FileSystem.get(conf);

    if (path.startsWith("hdfs://")) {
        throw new IOException("Path should not include 'hdfs://host:port'");
    }
    if (env != null) {
        try {
            path = env.substitute(path);
        } catch (Exception ex) {
        }
    }
    // if (!path.startsWith("/")) {
    // path = "/" + path;
    // }

    // We know that all job-specific jars are installed in the user's home
    // directory
    Path destPath = new Path(path);
    String userHome = fs.getHomeDirectory().toString();
    String absolutePath = userHome + "/" + destPath.toString();
    if (absolutePath.startsWith("hdfs://")) {
        // strip this off - for some reason under CDH4
        // DistributedCache.addFileToClassPath() keeps the hdfs:// part
        // of the URL in the classpath spec! Apache does not do this.
        absolutePath = absolutePath.replace("hdfs://", "");
        absolutePath = absolutePath.substring(absolutePath.indexOf("/"), absolutePath.length());
    }
    destPath = new Path(absolutePath);

    DistributedCache.addFileToClassPath(destPath, conf, fs);

    checkForWindowsAccessingHadoopOnLinux(conf);
}

---

From source file:distributed.hadoop.HDFSUtils.java

License:Open Source License

/**
 * Adds a file to the distributed cache for the supplied Configuration
 * /*from   w  w w.  j av  a 2  s.  c  om*/
 * @param hdfsConfig the hdfs configuration to use
 * @param conf the job configuration to configure
 * @param path the path to the file to add. This can be a local file, in which
 *          case it is first staged in HDFS, or a file in HDFS.
 * @param env environment variables
 * @return the filename only part of the path (this is what will be accessible
 *         from the distributed cache to a client via standard Java file IO)
 * @throws IOException if a problem occurs
 */
public static String addFileToDistributedCache(HDFSConfig hdfsConfig, Configuration conf, String path,
        Environment env) throws IOException {

    createTmpDistributedCacheDirIfNecessary(hdfsConfig);

    // conf.set(HDFSConfig.FS_DEFAULT_NAME,
    // HDFSConfig.constructHostURL(hdfsConfig, env));
    hdfsConfig.configureForHadoop(conf, env);

    // FileSystem fs = FileSystem.get(conf);

    if (path.startsWith("hdfs://")) {
        throw new IOException("Path should not include 'hdfs://host:port'");
    }
    if (env != null) {
        try {
            path = env.substitute(path);
        } catch (Exception ex) {
        }
    }

    File local = new File(path);
    if (local.exists()) {
        String fileNameOnly = "";
        if (path.lastIndexOf(File.separator) >= 0) {
            fileNameOnly = path.substring(path.lastIndexOf(File.separator) + File.separator.length(),
                    path.length());
        } else {
            fileNameOnly = path;
        }

        // copy the local file into HDFS staging
        path = WEKA_TEMP_DISTRIBUTED_CACHE_FILES + fileNameOnly;
        copyToHDFS(local.toString(), path, hdfsConfig, env, true);
    }

    // if (!path.startsWith("/")) {
    // path = "/" + path;
    // }

    String fileNameOnly = path.substring(path.lastIndexOf('/') + 1, path.length());

    try {
        DistributedCache.addCacheFile(new URI(resolvePath(path + "#" + fileNameOnly, env)), conf);
    } catch (URISyntaxException e) {
        throw new IOException(e);
    }
    DistributedCache.createSymlink(conf);

    return fileNameOnly;
}

---

From source file:distributed.hadoop.MapReduceJobConfig.java

License:Open Source License

/**
 * Substitute environment variables in a given string
 * //from ww w.j  a va  2 s.  c om
 * @param s the string to substitute in
 * @param env environment variables
 * @return the string with variables replaced
 */
protected static String environmentSubstitute(String s, Environment env) {
    if (env != null) {
        try {
            s = env.substitute(s);
        } catch (Exception ex) {
        }
    }

    return s;
}

---