Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path)

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:org.apache.sqoop.mapreduce.odps.HdfsOdpsImportJob.java

License:Apache License

@Override
protected void configureInputFormat(Job job, String tableName, String tableClassName, String splitByCol)
        throws ClassNotFoundException, IOException {
    fileType = getInputFileType();// w  ww  .  j  a v  a 2s  . c  o m

    super.configureInputFormat(job, tableName, tableClassName, splitByCol);

    if (isHCatJob) {
        SqoopHCatUtilities.configureExportInputFormat(options, job, context.getConnManager(), tableName,
                job.getConfiguration());
        return;
    } else if (fileType == FileType.AVRO_DATA_FILE) {
        LOG.debug("Configuring for Avro export");
        configureGenericRecordExportInputFormat(job, tableName);
    } else if (fileType == FileType.PARQUET_FILE) {
        LOG.debug("Configuring for Parquet export");
        configureGenericRecordExportInputFormat(job, tableName);
        FileSystem fs = FileSystem.get(job.getConfiguration());
        String uri = "dataset:" + fs.makeQualified(getInputPath());
        Exception caughtException = null;
        try {
            DatasetKeyInputFormat.configure(job).readFrom(uri);
        } catch (DatasetNotFoundException e) {
            LOG.warn(e.getMessage(), e);
            LOG.warn("Trying to get data schema from parquet file directly");
            caughtException = e;
        }
        if (caughtException != null && caughtException instanceof DatasetNotFoundException) {
            DatasetDescriptor descriptor = getDatasetDescriptorFromParquetFile(job, fs, uri);
            Dataset dataset = Datasets.create(uri, descriptor, GenericRecord.class);
            DatasetKeyInputFormat.configure(job).readFrom(dataset);
        }
    }

    FileInputFormat.addInputPath(job, getInputPath());
}

From source file:org.apache.sqoop.mapreduce.odps.HdfsOdpsImportJob.java

License:Apache License

private DatasetDescriptor getDatasetDescriptorFromParquetFile(Job job, FileSystem fs, String uri)
        throws IOException {

    ArrayList<FileStatus> files = new ArrayList<FileStatus>();
    FileStatus[] dirs;//from   w  w w. ja  v a2s  .  com
    dirs = fs.globStatus(fs.makeQualified(getInputPath()));
    for (int i = 0; (dirs != null && i < dirs.length); i++) {
        files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath(), HIDDEN_FILES_PATH_FILTER)));
        // We only check one file, so exit the loop when we have at least
        // one.
        if (files.size() > 0) {
            break;
        }
    }

    ParquetMetadata parquetMetadata;
    try {
        parquetMetadata = ParquetFileReader.readFooter(job.getConfiguration(),
                fs.makeQualified(files.get(0).getPath()));
    } catch (IOException e) {
        LOG.error("Wrong file format. Please check the export file's format.", e);
        throw e;
    }
    MessageType schema = parquetMetadata.getFileMetaData().getSchema();
    Schema avroSchema = new AvroSchemaConverter().convert(schema);
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder().schema(avroSchema).format(Formats.PARQUET)
            .compressionType(ParquetJob.getCompressionType(job.getConfiguration())).build();
    return descriptor;
}

From source file:org.apache.tajo.HBaseTestClusterUtil.java

License:Apache License

/**
 * Returns the path to the default root dir the minicluster uses.
 * Note: this does not cause the root dir to be created.
 * @return Fully qualified path for the default hbase root dir
 * @throws java.io.IOException//w w w.ja va  2  s .  c om
 */
public Path getDefaultRootDirPath() throws IOException {
    FileSystem fs = FileSystem.get(this.conf);
    return new Path(fs.makeQualified(fs.getHomeDirectory()), "hbase");
}

From source file:org.apache.tajo.storage.raw.TestDirectRawFile.java

License:Apache License

public Path getTestDir(FileSystem fs, String dir) throws IOException {
    Path path = new Path(dir);
    if (fs.exists(path))
        fs.delete(path, true);//from  w w w .j  a v  a 2 s .  c o  m

    fs.mkdirs(path);

    return fs.makeQualified(path);
}

From source file:org.apache.tajo.util.CommonTestingUtil.java

License:Apache License

/**
 *
 * @param dir a local directory to be created
 * @return  the created path// w  w w  .  j  a v  a2 s .  co  m
 * @throws java.io.IOException
 */
public static Path getTestDir(String dir) throws IOException {
    Path path = new Path(dir);
    FileSystem fs = FileSystem.getLocal(new Configuration());
    cleanupTestDir(dir);
    fs.mkdirs(path);

    return fs.makeQualified(path);
}

From source file:org.apache.tajo.util.CommonTestingUtil.java

License:Apache License

public static Path getTestDir() throws IOException {
    String randomStr = UUID.randomUUID().toString();
    Path path = new Path("target/test-data", randomStr);
    FileSystem fs = FileSystem.getLocal(new Configuration());
    if (fs.exists(path)) {
        fs.delete(path, true);// w w  w. java2 s.  com
    }

    fs.mkdirs(path);

    return fs.makeQualified(path);
}

From source file:org.apache.tajo.worker.LegacyTaskImpl.java

License:Apache License

@Override
public void init() throws IOException {
    initPlan();/*from w w w  .  j  av  a2  s.  c  om*/
    startScriptExecutors();

    if (context.getState() == TaskAttemptState.TA_PENDING) {
        // initialize a task temporal dir
        FileSystem localFS = executionBlockContext.getLocalFS();
        localFS.mkdirs(taskDir);

        if (request.getFetches().size() > 0) {
            inputTableBaseDir = localFS.makeQualified(executionBlockContext.getLocalDirAllocator()
                    .getLocalPathForWrite(getTaskAttemptDir(context.getTaskId()).toString(), systemConf));
            localFS.mkdirs(inputTableBaseDir);
            Path tableDir;
            for (String inputTable : context.getInputTables()) {
                tableDir = new Path(inputTableBaseDir, inputTable);
                if (!localFS.exists(tableDir)) {
                    LOG.info("the directory is created  " + tableDir.toUri());
                    localFS.mkdirs(tableDir);
                }
            }
        }
        // for localizing the intermediate data
        fetcherRunners.addAll(getFetchRunners(context, request.getFetches()));
    }
}

From source file:org.apache.tajo.worker.RangeRetrieverHandler.java

License:Apache License

public RangeRetrieverHandler(File outDir, Schema schema, BaseTupleComparator comp) throws IOException {
    this.file = outDir;
    BSTIndex index = new BSTIndex(new TajoConf());
    this.schema = schema;
    this.comp = comp;
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path indexPath = fs.makeQualified(new Path(outDir.getCanonicalPath(), "index"));
    this.idxReader = index.getIndexReader(indexPath, this.schema, this.comp);
    this.idxReader.open();
    LOG.info("BSTIndex is loaded from disk (" + idxReader.getFirstKey() + ", " + idxReader.getLastKey());
    this.decoder = RowStoreUtil.createDecoder(schema);
}

From source file:org.apache.tajo.worker.Task.java

License:Apache License

public void init() throws IOException {
    initPlan();/*from   w  ww  . j  ava 2 s .  com*/

    if (context.getState() == TaskAttemptState.TA_PENDING) {
        // initialize a task temporal dir
        FileSystem localFS = executionBlockContext.getLocalFS();
        localFS.mkdirs(taskDir);

        if (request.getFetches().size() > 0) {
            inputTableBaseDir = localFS.makeQualified(executionBlockContext.getLocalDirAllocator()
                    .getLocalPathForWrite(getTaskAttemptDir(context.getTaskId()).toString(), systemConf));
            localFS.mkdirs(inputTableBaseDir);
            Path tableDir;
            for (String inputTable : context.getInputTables()) {
                tableDir = new Path(inputTableBaseDir, inputTable);
                if (!localFS.exists(tableDir)) {
                    LOG.info("the directory is created  " + tableDir.toUri());
                    localFS.mkdirs(tableDir);
                }
            }
        }
        // for localizing the intermediate data
        fetcherRunners.addAll(getFetchRunners(context, request.getFetches()));
    }
}

From source file:org.apache.tajo.worker.TaskImpl.java

License:Apache License

@Override
public void init() throws IOException {
    LOG.info("Initializing: " + getId());

    initPlan();//w w  w  .ja  va 2 s  .  co m
    startScriptExecutors();

    if (context.getState() == TaskAttemptState.TA_PENDING) {
        // initialize a task temporal dir
        FileSystem localFS = executionBlockContext.getLocalFS();
        localFS.mkdirs(taskDir);

        if (request.getFetches().size() > 0) {
            inputTableBaseDir = localFS.makeQualified(executionBlockContext.getLocalDirAllocator()
                    .getLocalPathForWrite(getTaskAttemptDir(context.getTaskId()).toString(), systemConf));
            localFS.mkdirs(inputTableBaseDir);
            Path tableDir;
            for (String inputTable : context.getInputTables()) {
                tableDir = new Path(inputTableBaseDir, inputTable);
                if (!localFS.exists(tableDir)) {
                    LOG.info("the directory is created  " + tableDir.toUri());
                    localFS.mkdirs(tableDir);
                }
            }
        }
        // for localizing the intermediate data
        fetcherRunners.addAll(getFetchRunners(context, request.getFetches()));
    }
}