Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:org.apache.hive.beeline.BeeLine.java

License:Apache License

private int executeFile(String fileName) {
    InputStream initStream = null;
    try {//from   w w  w  .  ja v a2  s. c  o  m
        if (!isBeeLine) {
            org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(fileName);
            FileSystem fs;
            HiveConf conf = getCommands().getHiveConf(true);
            if (!path.toUri().isAbsolute()) {
                fs = FileSystem.getLocal(conf);
                path = fs.makeQualified(path);
            } else {
                fs = FileSystem.get(path.toUri(), conf);
            }
            initStream = fs.open(path);
        } else {
            initStream = new FileInputStream(fileName);
        }
        return execute(getConsoleReader(initStream), !getOpts().getForce());
    } catch (Throwable t) {
        handleException(t);
        return ERRNO_OTHER;
    } finally {
        IOUtils.closeStream(initStream);
        consoleReader = null;
        output(""); // dummy new line
    }
}

---

From source file:org.apache.hive.hcatalog.mapreduce.HCatBaseInputFormat.java

License:Apache License

private List<String> setInputPath(JobConf jobConf, String location) throws IOException {

    // ideally we should just call FileInputFormat.setInputPaths() here - but
    // that won't work since FileInputFormat.setInputPaths() needs
    // a Job object instead of a JobContext which we are handed here

    int length = location.length();
    int curlyOpen = 0;
    int pathStart = 0;
    boolean globPattern = false;
    List<String> pathStrings = new ArrayList<String>();

    for (int i = 0; i < length; i++) {
        char ch = location.charAt(i);
        switch (ch) {
        case '{': {
            curlyOpen++;// ww w  .j  a  v  a2  s  .c o  m
            if (!globPattern) {
                globPattern = true;
            }
            break;
        }
        case '}': {
            curlyOpen--;
            if (curlyOpen == 0 && globPattern) {
                globPattern = false;
            }
            break;
        }
        case ',': {
            if (!globPattern) {
                pathStrings.add(location.substring(pathStart, i));
                pathStart = i + 1;
            }
            break;
        }
        }
    }
    pathStrings.add(location.substring(pathStart, length));

    String separator = "";
    StringBuilder str = new StringBuilder();

    boolean ignoreInvalidPath = jobConf.getBoolean(HCatConstants.HCAT_INPUT_IGNORE_INVALID_PATH_KEY,
            HCatConstants.HCAT_INPUT_IGNORE_INVALID_PATH_DEFAULT);
    Iterator<String> pathIterator = pathStrings.iterator();
    while (pathIterator.hasNext()) {
        String pathString = pathIterator.next();
        if (ignoreInvalidPath && org.apache.commons.lang.StringUtils.isBlank(pathString)) {
            continue;
        }
        Path path = new Path(pathString);
        FileSystem fs = path.getFileSystem(jobConf);
        if (ignoreInvalidPath && !fs.exists(path)) {
            pathIterator.remove();
            continue;
        }
        final String qualifiedPath = fs.makeQualified(path).toString();
        str.append(separator).append(StringUtils.escapeString(qualifiedPath));
        separator = StringUtils.COMMA_STR;
    }

    if (!ignoreInvalidPath || !pathStrings.isEmpty()) {
        jobConf.set("mapred.input.dir", str.toString());
    }
    return pathStrings;
}

---

From source file:org.apache.ignite.yarn.utils.IgniteYarnUtils.java

License:Apache License

/**
 * @param file Path.//from  w w w  .  j  a v a 2s. c o m
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);

    file = fs.makeQualified(file);

    FileStatus stat = fs.getFileStatus(file);

    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);

    return resource;
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Return true iff the path is on the given filesystem.
 *///from ww  w  .j  ava  2s.  c  o m
public static boolean isPathOnFileSystem(Path path, FileSystem fs) {
    try {
        // Call makeQualified() for the side-effect of FileSystem.checkPath() which will
        // throw an exception if path is not on fs.
        fs.makeQualified(path);
        return true;
    } catch (IllegalArgumentException e) {
        // Path is not on fs.
        return false;
    }
}

---

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Adds an input path to the job configuration
 * //  w  w w.j a  v a2  s.  c o  m
 * @param f
 *            File
 * @param config
 *            Configuration
 * @param job
 *            Job
 * @throws IOException
 */
protected void addInputPath(File f, Configuration config, Job job) throws IOException {
    FileSystem fs = FileSystem.getLocal(config);
    Path inputPath = fs.makeQualified(new Path(f.getAbsolutePath()));
    FileInputFormat.addInputPath(job, inputPath);
}

---

From source file:org.apache.jena.hadoop.rdf.io.output.AbstractNodeTupleOutputFormatTests.java

License:Apache License

/**
 * Adds an output path to the job configuration
 * //from  w w w . j a  v  a2s. c  o m
 * @param f
 *            File
 * @param config
 *            Configuration
 * @param job
 *            Job
 * @throws IOException
 */
protected void addOutputPath(File f, Configuration config, Job job) throws IOException {
    FileSystem fs = FileSystem.getLocal(config);
    Path outputPath = fs.makeQualified(new Path(f.getAbsolutePath()));
    FileOutputFormat.setOutputPath(job, outputPath);
}

---

From source file:org.apache.kylin.common.KylinConfigBase.java

License:Apache License

public String getHdfsWorkingDirectory() {
    if (cachedHdfsWorkingDirectory != null)
        return cachedHdfsWorkingDirectory;

    String root = getOptional("kylin.env.hdfs-working-dir", "/kylin");

    Path path = new Path(root);
    if (!path.isAbsolute())
        throw new IllegalArgumentException("kylin.env.hdfs-working-dir must be absolute, but got " + root);

    // make sure path is qualified
    try {//from  w ww  .jav a2  s  .  c  om
        FileSystem fs = path.getFileSystem(HadoopUtil.getCurrentConfiguration());
        path = fs.makeQualified(path);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    // append metadata-url prefix
    root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString();

    if (!root.endsWith("/"))
        root += "/";

    cachedHdfsWorkingDirectory = root;
    if (cachedHdfsWorkingDirectory.startsWith("file:")) {
        cachedHdfsWorkingDirectory = cachedHdfsWorkingDirectory.replace("file:", "file://");
    } else if (cachedHdfsWorkingDirectory.startsWith("maprfs:")) {
        cachedHdfsWorkingDirectory = cachedHdfsWorkingDirectory.replace("maprfs:", "maprfs://");
    }
    return cachedHdfsWorkingDirectory;
}

---

From source file:org.apache.kylin.source.hive.ITHiveTezUnionAllTest.java

License:Apache License

private void testMaterializeView(boolean isDistributeBy) throws Exception {
    KylinConfig config = getTestConfig();

    /**/*  ww w  . j a  v a2s  . co  m*/
     * For UT debug
     * config.setProperty("kylin.job.use-remote-cli", "true");
     */

    String viewName = "test_union_all_view";
    String tableName = "test_union_all_table";

    HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
    JobEngineConfig jobConf = new JobEngineConfig(config);
    String storagePath = JobBuilderSupport.getJobWorkingDir(jobConf, "it-test") + "/" + tableName;

    StringBuilder testCmd = new StringBuilder();
    testCmd.append("USE " + config.getHiveDatabaseForIntermediateTable() + ";").append("\n");
    testCmd.append("SET hive.execution.engine=tez;");
    testCmd.append("DROP VIEW IF EXISTS " + viewName + ";\n");
    testCmd.append("CREATE VIEW " + viewName
            + " AS SELECT * FROM test_kylin_fact UNION ALL SELECT * FROM test_kylin_fact").append(";\n");
    testCmd.append("DROP TABLE IF EXISTS " + tableName + ";\n");
    testCmd.append("CREATE TABLE IF NOT EXISTS " + tableName + "\n");
    testCmd.append("LOCATION '" + storagePath + "'\n");
    testCmd.append("AS SELECT * FROM " + viewName + "\n");
    if (isDistributeBy)
        hiveCmdBuilder.addStatementWithRedistributeBy(testCmd);
    else
        hiveCmdBuilder.addStatement(testCmd.toString());

    Path rootPath = new Path(storagePath);
    FileSystem fs = HadoopUtil.getFileSystem(storagePath);

    fs.delete(rootPath, true);
    fs.mkdirs(rootPath);

    config.getCliCommandExecutor().execute(hiveCmdBuilder.build());

    rootPath = fs.makeQualified(rootPath);
    for (FileStatus statsFolder : fs.listStatus(rootPath)) {
        if (isDistributeBy)
            Assert.assertTrue(!statsFolder.isDirectory());
        else
            Assert.assertTrue(statsFolder.isDirectory());
    }

    HiveCmdBuilder cleanupCmdBuilder = new HiveCmdBuilder();
    StringBuilder cleanupCmd = new StringBuilder();
    cleanupCmd.append("USE " + config.getHiveDatabaseForIntermediateTable() + ";").append("\n");
    cleanupCmd.append("DROP VIEW IF EXISTS " + viewName + ";\n");
    cleanupCmd.append("DROP TABLE IF EXISTS " + tableName + ";\n");
    cleanupCmdBuilder.addStatement(cleanupCmd.toString());
    config.getCliCommandExecutor().execute(cleanupCmdBuilder.build());
    fs.delete(rootPath, true);
}

---

From source file:org.apache.kylin.storage.hbase.HBaseConnection.java

License:Apache License

public static String makeQualifiedPathInHBaseCluster(String inPath) {
    Path path = new Path(inPath);
    path = Path.getPathWithoutSchemeAndAuthority(path);

    try {/* w  w w.  j  av a  2s. c  o m*/
        FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
        return fs.makeQualified(path).toString();
    } catch (IOException e) {
        throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e);
    }
}

---

From source file:org.apache.kylin.storage.hbase.steps.HFileOutputFormat3.java

License:Apache License

/**
 * Configure <code>job</code> with a TotalOrderPartitioner, partitioning against
 * <code>splitPoints</code>. Cleans up the partitions file after job exists.
 *///from  w w  w .  j  av  a  2 s .  c  o  m
static void configurePartitioner(Job job, List<ImmutableBytesWritable> splitPoints) throws IOException {
    Configuration conf = job.getConfiguration();
    // create the partitions file
    FileSystem fs = FileSystem.get(conf);
    Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + RandomUtil.randomUUID());
    fs.makeQualified(partitionsPath);
    writePartitions(conf, partitionsPath, splitPoints);
    fs.deleteOnExit(partitionsPath);

    // configure job to use it
    job.setPartitionerClass(TotalOrderPartitioner.class);
    TotalOrderPartitioner.setPartitionFile(conf, partitionsPath);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »