Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException 

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.asakusafw.testdriver.file.FileExporterRetriever.java

License:Apache License

@Override
public void truncate(FileExporterDescription description, TestContext context) throws IOException {
    LOG.info("cleaning output files: {}", description);
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    FileSystem fs = path.getFileSystem(config);
    Path output = path.getParent();
    Path target;//from w  w  w.j a v  a 2 s .c om
    if (output == null) {
        LOG.warn(
                "?????????: {}",
                path);
        target = fs.makeQualified(path);
    } else {
        LOG.warn("??????: {}", output);
        target = fs.makeQualified(output);
    }
    LOG.debug("start removing file: {}", target);
    boolean succeed = fs.delete(target, true);
    LOG.debug("finish removing file (succeed={}): {}", succeed, target);
}

---

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryDataModelSource.java

License:Apache License

/**
 * Creates a new instance./*  w  w w  .ja  v a2 s.  c o  m*/
 * @param conf current configuration
 * @param definition data type
 * @param pathExpression the source path (can include wildcard)
 * @throws IOException if failed to create instance
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public TemporaryDataModelSource(Configuration conf, DataModelDefinition<?> definition, String pathExpression)
        throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (definition == null) {
        throw new IllegalArgumentException("definition must not be null"); //$NON-NLS-1$
    }
    if (pathExpression == null) {
        throw new IllegalArgumentException("pathExpression must not be null"); //$NON-NLS-1$
    }
    this.conf = conf;
    this.definition = (DataModelDefinition<Object>) definition;
    this.object = definition.toObject(definition.newReflection().build());
    Path path = new Path(pathExpression);
    this.fs = path.getFileSystem(conf);
    FileStatus[] list = fs.globStatus(path);
    List<Path> paths = new ArrayList<>();
    for (int i = 0; i < list.length; i++) {
        paths.add(list[i].getPath());
    }
    this.rest = paths.iterator();
}

---

From source file:com.asakusafw.testdriver.temporary.TemporaryDataModelSource.java

License:Apache License

/**
 * Creates a new instance.// w  w  w  .  j  a  v  a 2s. c  o m
 * @param conf current configuration
 * @param definition data type
 * @param pathExpression the source path (can include wildcard)
 * @throws IOException if failed to create instance
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public TemporaryDataModelSource(Configuration conf, DataModelDefinition<?> definition, String pathExpression)
        throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (definition == null) {
        throw new IllegalArgumentException("definition must not be null"); //$NON-NLS-1$
    }
    if (pathExpression == null) {
        throw new IllegalArgumentException("pathExpression must not be null"); //$NON-NLS-1$
    }
    this.conf = conf;
    this.definition = (DataModelDefinition<Object>) definition;
    this.object = definition.toObject(definition.newReflection().build());
    Path path = new Path(pathExpression);
    this.fs = path.getFileSystem(conf);
    FileStatus[] list = fs.globStatus(path);
    List<Path> paths = Lists.create();
    for (int i = 0; i < list.length; i++) {
        paths.add(list[i].getPath());
    }
    this.rest = paths.iterator();
}

---

From source file:com.asakusafw.workflow.hadoop.HadoopDelete.java

License:Apache License

private static void delete(Configuration conf, Path path) throws IOException {
    FileSystem fs = path.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug("deleting file: {}", fs.makeQualified(path));
    }/*ww  w  .  j a  v a2  s .c o  m*/
    boolean deleted = fs.delete(path, true);
    if (LOG.isDebugEnabled()) {
        if (deleted) {
            LOG.debug("delete success: {}", fs.makeQualified(path));
        } else if (fs.exists(path)) {
            LOG.debug("delete failed: {}", fs.makeQualified(path));
        } else {
            LOG.debug("target file is not found: {}", fs.makeQualified(path));
        }
    }
}

---

From source file:com.ashishpaliwal.hadoop.utils.inputformat.CsvRecordReader.java

License:Apache License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;

    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt(MAX_LINE_LENGTH, 2147483647);
    this.start = split.getStart();
    this.end = (this.start + split.getLength());
    Path file = split.getPath();
    this.compressionCodecs = new CompressionCodecFactory(job);
    this.codec = this.compressionCodecs.getCodec(file);

    FileSystem fs = file.getFileSystem(job);
    this.fileIn = fs.open(file);
    if (isCompressedInput()) {
        this.decompressor = CodecPool.getDecompressor(this.codec);
        if ((this.codec instanceof SplittableCompressionCodec)) {
            SplitCompressionInputStream cIn = ((SplittableCompressionCodec) this.codec).createInputStream(
                    this.fileIn, this.decompressor, this.start, this.end,
                    SplittableCompressionCodec.READ_MODE.BYBLOCK);

            this.in = new CsvLineReader(cIn, job);
            this.start = cIn.getAdjustedStart();
            this.end = cIn.getAdjustedEnd();
            this.filePosition = cIn;
        } else {/*from  w w  w . j a v  a 2  s .  c  o  m*/
            this.in = new CsvLineReader(this.codec.createInputStream(this.fileIn, this.decompressor), job);
            this.filePosition = this.fileIn;
        }
    } else {
        this.fileIn.seek(this.start);
        this.in = new CsvLineReader(this.fileIn, job);
        this.filePosition = this.fileIn;
    }

    if (this.start != 0L) {
        this.start += this.in.readLine(new Text(), 0, maxBytesToConsume(this.start));
    }
    this.pos = this.start;
}

---

From source file:com.awcoleman.ExampleJobSummaryLogWithOutput.BinRecRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit insplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    FileSplit split = (FileSplit) insplit;

    start = split.getStart();//from   w ww  . j a  v a 2s  .c  o  m
    end = start + split.getLength();
    pos = start;

    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);
    fsin = fs.open(path);
}

---

From source file:com.awcoleman.ExampleJobSummaryLogWithOutput.BinRecToAvroRecDriver.java

License:Apache License

public int run(String[] args) throws Exception {

    String input = null;/*from   ww  w. j a  v  a 2 s.c  om*/
    String output = null;

    if (args.length < 2) {
        System.err.printf("Usage: %s <input> <output>\n", this.getClass().getSimpleName());
        return -1;
    } else {
        input = args[0];
        output = args[1];
    }

    Job job = Job.getInstance(getConf(), "BinRecToAvroRecDriver");
    Configuration conf = job.getConfiguration();

    //Add job log to hold Driver logging (and any summary info about the dataset,job, or counters we want to write)
    String fapath = createTempFileAppender(job);

    //get schema
    Schema outSchema = ReflectData.get().getSchema(com.awcoleman.examples.avro.BinRecForPartitions.class);
    job.getConfiguration().set("outSchema", outSchema.toString());

    //Job conf settings
    job.setJarByClass(BinRecToAvroRecDriver.class);
    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);
    job.setInputFormatClass(BinRecInputFormat.class);
    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    AvroJob.setOutputKeySchema(job, outSchema);

    AvroJob.setMapOutputKeySchema(job, Schema.create(Schema.Type.STRING));
    AvroJob.setMapOutputValueSchema(job, outSchema);

    //Job output compression
    FileOutputFormat.setCompressOutput(job, true);
    job.getConfiguration().set(AvroJob.CONF_OUTPUT_CODEC, DataFileConstants.DEFLATE_CODEC);

    //Input and Output Paths
    FileInputFormat.setInputPaths(job, new Path(input));
    Path outPath = new Path(output);
    FileOutputFormat.setOutputPath(job, outPath);
    outPath.getFileSystem(conf).delete(outPath, true);

    boolean jobCompletionStatus = job.waitForCompletion(true);

    //Print Custom Counters before exiting
    Counters counters = job.getCounters();
    for (MYJOB_CNTRS customCounter : MYJOB_CNTRS.values()) {
        Counter thisCounter = counters.findCounter(customCounter);
        System.out.println("Custom Counter " + customCounter + "=" + thisCounter.getValue());
    }

    long mycnt1 = job.getCounters()
            .findCounter("com.awcoleman.TestingGettingContainerLogger.BinRecToAvroRecDriver$MYJOB_CNTRS",
                    "MYCNT1")
            .getValue();
    long mycnt2 = job.getCounters()
            .findCounter("com.awcoleman.TestingGettingContainerLogger.BinRecToAvroRecDriver$MYJOB_CNTRS",
                    "MYCNT2")
            .getValue();
    long mycnt3 = job.getCounters()
            .findCounter("com.awcoleman.TestingGettingContainerLogger.BinRecToAvroRecDriver$MYJOB_CNTRS",
                    "MYCNT3")
            .getValue();

    long myfakekpi = mycnt1 - mycnt2;

    String msgMyfakekpi = "The Fake KPI of the Dataset: " + String.format("%,d", myfakekpi);
    System.out.println(msgMyfakekpi);
    logger.info(msgMyfakekpi);

    //Finished, so move job log to HDFS in _log dir, clean
    copyTempFileAppenderToHDFSOutpath(job, fapath, output);

    return jobCompletionStatus ? 0 : 1;
}

---

From source file:com.bah.culvert.hive.CulvertHiveUtils.java

License:Apache License

/**
 * Get the culvert configuration. Checks to see if we should just treat the
 * properties as the configuration, or if we should load an external file.
 * Finally, returns the configuration based off of the properties.
 * //from   w  w w . j a v a  2 s  .  c  om
 * @param props The properties to examine and possibly turn into a
 *        configuration.
 * @return The new configuration.
 * @see CulvertHiveUtils#isCulvertConfigurationEmbedded(Properties)
 * @see #propsToConf(Properties)
 * @see #setCulvertConfiguration(Properties, Configuration)
 * @see #setCulvertConfiguration(Properties, String)
 */
public static Configuration getCulvertConfiguration(Properties props) {
    if (isCulvertConfigurationEmbedded(props)) {

        return propsToConf(props);
    } else {
        String fileName = props.getProperty(CULVERT_HIVE_EXTERNAL_CONF_FILE_CONF_KEY);
        Path fileLocation = new Path(fileName);
        FSDataInputStream input = null;
        try {
            FileSystem fs = fileLocation.getFileSystem(new Configuration());
            Configuration conf = new Configuration(false);
            input = fs.open(fileLocation);
            conf.readFields(input);
            return conf;
        } catch (IOException e) {
            throw new RuntimeException(e);
        } finally {
            if (input != null)
                try {
                    input.close();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
        }
    }
}

---

From source file:com.bah.lucene.hdfs.HdfsDirectory.java

License:Apache License

public HdfsDirectory(Configuration configuration, Path path) throws IOException {
    this._path = path;
    _fileSystem = path.getFileSystem(configuration);
    _fileSystem.mkdirs(path);//from w ww .j a v a 2s  .com
    setLockFactory(NoLockFactory.getNoLockFactory());
}

---

From source file:com.bah.lucene.hdfs.SoftlinkHdfsDirectory.java

License:Apache License

/**
 * Creates a new SoftlinkHdfsDirectory./*  w  w  w  .java  2  s  . c  o  m*/
 * 
 * @param configuration
 *          the {@link Configuration} object.
 * @param storePath
 *          the path where the data is actually stored.
 * @param linkPath
 *          the path where the links are stored.
 * @throws IOException
 */
public SoftlinkHdfsDirectory(Configuration configuration, Path storePath, Path linkPath) throws IOException {
    super(configuration, linkPath);
    FileSystem fileSystem = storePath.getFileSystem(configuration);
    _storePath = fileSystem.makeQualified(storePath);
    _linkPath = fileSystem.makeQualified(linkPath);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »