Example usage for org.apache.hadoop.fs Path makeQualified

List of usage examples for org.apache.hadoop.fs Path makeQualified

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path makeQualified.

Prototype

@Deprecated
public Path makeQualified(FileSystem fs)

Source Link

Document

Returns a qualified path object for the FileSystem 's working directory.

Usage

From source file:org.apache.lens.driver.hive.HiveDriver.java

License:Apache License

/**
 * Adds the persistent path./* w  ww.  j  av a 2 s  . c om*/
 *
 * @param context the context
 * @throws IOException Signals that an I/O exception has occurred.
 */
void addPersistentPath(QueryContext context) throws IOException {
    String hiveQuery;
    Configuration qdconf = context.getDriverConf(this);
    boolean addInsertOverwrite = qdconf.getBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE,
            LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE);
    if (context.isDriverPersistent() && addInsertOverwrite
            && (context.getSelectedDriverQuery().startsWith("SELECT")
                    || context.getSelectedDriverQuery().startsWith("select"))) {
        // store persistent data into user specified location
        // If absent, take default home directory
        Path resultSetPath = context.getHDFSResultDir();
        // create query
        StringBuilder builder = new StringBuilder("INSERT OVERWRITE DIRECTORY ");
        context.setDriverResultPath(
                resultSetPath.makeQualified(resultSetPath.getFileSystem(context.getConf())).toString());
        builder.append('"').append(resultSetPath).append("\" ");
        String outputDirFormat = qdconf.get(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT);
        if (outputDirFormat != null) {
            builder.append(outputDirFormat);
        }
        builder.append(' ').append(context.getSelectedDriverQuery()).append(' ');
        hiveQuery = builder.toString();
    } else {
        context.unSetDriverPersistent();
        hiveQuery = context.getSelectedDriverQuery();
    }
    log.info("Hive driver {} query:{}", getFullyQualifiedName(), hiveQuery);
    context.setSelectedDriverQuery(hiveQuery);
}

From source file:org.apache.lens.lib.query.TestAbstractFileFormatter.java

License:Apache License

/**
 * Test formatter./* w ww.  ja  v  a  2 s .  c  o  m*/
 *
 * @param conf            the conf
 * @param charsetEncoding the charset encoding
 * @param outputParentDir the output parent dir
 * @param fileExtn        the file extn
 * @param columnNames     the column names
 * @throws IOException Signals that an I/O exception has occurred.
 */
protected void testFormatter(Configuration conf, String charsetEncoding, String outputParentDir,
        String fileExtn, LensResultSetMetadata columnNames) throws IOException {

    QueryContext ctx = createContext(conf, null);

    Path expectedFinalPath = new Path(outputParentDir, ctx.getQueryHandle() + fileExtn);
    FileSystem fs = expectedFinalPath.getFileSystem(conf);
    expectedFinalPath = expectedFinalPath.makeQualified(fs);
    validateFormatter(conf, charsetEncoding, outputParentDir, fileExtn, columnNames, ctx, expectedFinalPath);
}

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test text files output path./* w w w  .j av a 2 s . c o  m*/
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileOutputPath() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.set("test.partfile.dir", partFileTextDir.toString());
    conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
    conf.set(LensConfConstants.QUERY_OUTPUT_HEADER,
            "firstcolsecondcolthirdcolfourthcolfifthcolsixthcolseventhcol");
    QueryContext ctx = createContext(conf, "test.Query_1 name");
    Path expectedFinalPath = new Path(LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT,
            "test.Query_1_name-" + ctx.getQueryHandle() + ".txt");
    FileSystem fs = expectedFinalPath.getFileSystem(conf);
    expectedFinalPath = expectedFinalPath.makeQualified(fs);
    validateFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
            getMockedResultSetWithoutComma(), ctx, expectedFinalPath);
}

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test text files with a long output path.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 *//*from  ww w .  j  a  v  a  2s  .  c  o  m*/
@Test
public void testTextFileLongOutputPath() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.set("test.partfile.dir", partFileTextDir.toString());
    conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
    conf.set(LensConfConstants.QUERY_OUTPUT_HEADER,
            "firstcolsecondcolthirdcolfourthcolfifthcolsixthcolseventhcol");
    QueryContext ctx = createContext(conf,
            "test-Query 1^name12345678901234567890123456789012345678901234567890");
    Path expectedFinalPath = new Path(LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT,
            "test-Query_1_name123456789012345678901234567890123-" + ctx.getQueryHandle() + ".txt");
    FileSystem fs = expectedFinalPath.getFileSystem(conf);
    expectedFinalPath = expectedFinalPath.makeQualified(fs);
    validateFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
            getMockedResultSetWithoutComma(), ctx, expectedFinalPath);
}

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test zip csv files output path.//from  w  w w  .j a v a  2  s .  c o m
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testCSVZipFileOutputPath() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
    conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
    QueryContext ctx = createContext(conf, "Test.query_1 name");
    Path expectedFinalPath = new Path(LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT,
            "Test.query_1_name-" + ctx.getQueryHandle() + ".zip");
    FileSystem fs = expectedFinalPath.getFileSystem(conf);
    expectedFinalPath = expectedFinalPath.makeQualified(fs);
    validateFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
            getMockedResultSetWithoutComma(), ctx, expectedFinalPath);
    ZipEntry ze = null;
    ZipInputStream zin = new ZipInputStream(fs.open(expectedFinalPath));
    int i = 0;
    while ((ze = zin.getNextEntry()) != null) {
        Assert.assertEquals(ze.getName(), "Test.query_1_name-" + ctx.getQueryHandle() + "_part-" + i + ".csv");
        i++;
        zin.closeEntry();
    }
    zin.close();
}

From source file:org.apache.mahout.common.HadoopUtil.java

License:Apache License

public static InputStream openStream(Path path, Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    return fs.open(path.makeQualified(fs));
}

From source file:org.apache.mahout.common.iterator.sequencefile.SequenceFileIterator.java

License:Apache License

/**
 * @throws IOException if path can't be read, or its key or value class can't be instantiated
 *//*from   w w w.  j a v a2s  .com*/

public SequenceFileIterator(Path path, boolean reuseKeyValueInstances, Configuration conf) throws IOException {
    key = null;
    value = null;
    FileSystem fs = path.getFileSystem(conf);
    path = path.makeQualified(fs);
    reader = new SequenceFile.Reader(fs, path, conf);
    this.conf = conf;
    keyClass = (Class<K>) reader.getKeyClass();
    valueClass = (Class<V>) reader.getValueClass();
    noValue = NullWritable.class.equals(valueClass);
    this.reuseKeyValueInstances = reuseKeyValueInstances;
}

From source file:org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterator.java

License:Apache License

/**
 * @throws IOException if path can't be read, or its key or value class can't be instantiated
 *///from ww w  .  j  av  a  2 s . c o  m

public SequenceFileValueIterator(Path path, boolean reuseKeyValueInstances, Configuration conf)
        throws IOException {
    value = null;
    FileSystem fs = path.getFileSystem(conf);
    path = path.makeQualified(fs);
    reader = new SequenceFile.Reader(fs, path, conf);
    this.conf = conf;
    Class<? extends Writable> keyClass = (Class<? extends Writable>) reader.getKeyClass();
    key = ReflectionUtils.newInstance(keyClass, conf);
    valueClass = (Class<V>) reader.getValueClass();
    this.reuseKeyValueInstances = reuseKeyValueInstances;
}

From source file:org.apache.nutch.admin.AdministrationApp.java

License:Apache License

/**
 * creates an instance object from a instance folder
 * //from  w  w w.  j a  v  a  2 s  .co m
 * @param defaultConf
 * @param folder
 * @return an instance representation of this folder
 * @throws IOException
 *           in case the folder is not a valid instance folder
 */
public static NutchInstance loadNutchInstance(Configuration defaultConf, Path folder) throws IOException {
    Path instanceConfFolder = new Path(folder, "conf");

    Configuration conf = NutchConfiguration.create();
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(instanceConfFolder) && fs.isDirectory(instanceConfFolder)) {
        Path instanceSiteConf = new Path(instanceConfFolder, "nutch-site.xml");
        if (fs.exists(instanceSiteConf)) {
            Configuration instanceConf = new Configuration(defaultConf);
            instanceConf.addResource(instanceSiteConf.makeQualified(fs));
            return new NutchInstance(folder.getName(), folder, instanceConf);
        }
    }
    throw new IOException("not a valid instance folder: " + folder);
}

From source file:org.apache.nutch.admin.GuiConfigUtil.java

License:Apache License

private static void copyConfigurationFiles(Path target) throws FileNotFoundException, IOException {
    FileSystem fs = FileSystem.get(NutchConfiguration.create());

    InputStream in = AdministrationApp.class.getResourceAsStream("/nutch-default.xml");
    OutputStream out = new FileOutputStream(
            new File(new File(target.makeQualified(fs).toUri()), "nutch-default.xml"));
    copyContent(in, out);//from   w w  w .j av  a 2  s. c om

    in = AdministrationApp.class.getResourceAsStream("/nutch-site.xml");
    out = new FileOutputStream(new File(new File(target.makeQualified(fs).toUri()), "nutch-site.xml"));
    copyContent(in, out);
}