Example usage for org.apache.hadoop.fs FileSystem getDefaultReplication

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getDefaultReplication.

Prototype

public short getDefaultReplication(Path path) 

Source Link

Document

Get the default replication for a path.

Usage

From source file:org.apache.orc.impl.PhysicalFsWriter.java

License:Apache License

public PhysicalFsWriter(FileSystem fs, Path path, OrcFile.WriterOptions opts) throws IOException {
    this.path = path;
    this.defaultStripeSize = this.adjustedStripeSize = opts.getStripeSize();
    this.addBlockPadding = opts.getBlockPadding();
    if (opts.isEnforceBufferSize()) {
        this.bufferSize = opts.getBufferSize();
    } else {/*from  ww  w  .  j  a  v  a 2  s .c o  m*/
        this.bufferSize = WriterImpl.getEstimatedBufferSize(defaultStripeSize,
                opts.getSchema().getMaximumId() + 1, opts.getBufferSize());
    }
    this.compress = opts.getCompress();
    this.paddingTolerance = opts.getPaddingTolerance();
    this.blockSize = opts.getBlockSize();
    LOG.info(
            "ORC writer created for path: {} with stripeSize: {} blockSize: {}"
                    + " compression: {} bufferSize: {}",
            path, defaultStripeSize, blockSize, compress, bufferSize);
    rawWriter = fs.create(path, false, HDFS_BUFFER_SIZE, fs.getDefaultReplication(path), blockSize);
    CompressionCodec codec = WriterImpl.createCodec(compress);
    writer = new OutStream("metadata", bufferSize, codec, new DirectStream(rawWriter));
    protobufWriter = CodedOutputStream.newInstance(writer);
}

---

From source file:org.apache.parquet.hadoop.ParquetFileWriter.java

License:Apache License

/**
 * @param configuration Hadoop configuration
 * @param schema the schema of the data/* ww w  . j  av a 2s .c o m*/
 * @param file the file to write to
 * @param mode file creation mode
 * @param rowGroupSize the row group size
 * @throws IOException if the file can not be created
 */
public ParquetFileWriter(Configuration configuration, MessageType schema, Path file, Mode mode,
        long rowGroupSize, int maxPaddingSize) throws IOException {
    TypeUtil.checkValidWriteSchema(schema);
    this.schema = schema;
    FileSystem fs = file.getFileSystem(configuration);
    boolean overwriteFlag = (mode == Mode.OVERWRITE);

    if (supportsBlockSize(fs)) {
        // use the default block size, unless row group size is larger
        long dfsBlockSize = Math.max(fs.getDefaultBlockSize(file), rowGroupSize);

        this.alignment = PaddingAlignment.get(dfsBlockSize, rowGroupSize, maxPaddingSize);
        this.out = fs.create(file, overwriteFlag, DFS_BUFFER_SIZE_DEFAULT, fs.getDefaultReplication(file),
                dfsBlockSize);

    } else {
        this.alignment = NoAlignment.get(rowGroupSize);
        this.out = fs.create(file, overwriteFlag);
    }
}

---

• «
• 1
• 2
• »