Example usage for org.apache.hadoop.fs FileSystem create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize,
        Progressable progress) throws IOException 

Source Link

Document

Create an FSDataOutputStream at the indicated Path with write-progress reporting.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private long copyToTmpFile(Path tmpTargetPath, FileSystem targetFS, FileStatus sourceFileStatus,
        Mapper.Context context, EnumSet<FileAttribute> fileAttributes, Map<Long, Long> received)
        throws IOException {
    OutputStream outStream = new BufferedOutputStream(targetFS.create(tmpTargetPath, true, BUFFER_SIZE,
            getReplicationFactor(fileAttributes, sourceFileStatus, targetFS),
            getBlockSize(fileAttributes, sourceFileStatus, targetFS), context));
    return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, context, received);
}

---

From source file:com.uber.hoodie.common.table.log.avro.AvroLogAppender.java

License:Apache License

public AvroLogAppender(HoodieLogAppendConfig config) throws IOException, InterruptedException {
    FileSystem fs = config.getFs();
    this.config = config;
    this.autoFlush = config.isAutoFlush();
    GenericDatumWriter<IndexedRecord> datumWriter = new GenericDatumWriter<>(config.getSchema());
    this.writer = new DataFileWriter<>(datumWriter);
    Path path = config.getLogFile().getPath();

    if (fs.exists(path)) {
        //TODO - check for log corruption and roll over if needed
        log.info(config.getLogFile() + " exists. Appending to existing file");
        // this log path exists, we will append to it
        fs = FileSystem.get(fs.getConf());
        try {//from   w w w  . j a  v  a 2s  .co  m
            this.output = fs.append(path, config.getBufferSize());
        } catch (RemoteException e) {
            // this happens when either another task executor writing to this file died or data node is going down
            if (e.getClassName().equals(AlreadyBeingCreatedException.class.getName())
                    && fs instanceof DistributedFileSystem) {
                log.warn("Trying to recover log on path " + path);
                if (FSUtils.recoverDFSFileLease((DistributedFileSystem) fs, path)) {
                    log.warn("Recovered lease on path " + path);
                    // try again
                    this.output = fs.append(path, config.getBufferSize());
                } else {
                    log.warn("Failed to recover lease on path " + path);
                    throw new HoodieException(e);
                }
            }
        }
        this.writer.appendTo(new AvroFSInput(FileContext.getFileContext(fs.getConf()), path), output);
        // we always want to flush to disk everytime a avro block is written
        this.writer.setFlushOnEveryBlock(true);
    } else {
        log.info(config.getLogFile() + " does not exist. Create a new file");
        this.output = fs.create(path, false, config.getBufferSize(), config.getReplication(),
                config.getBlockSize(), null);
        this.writer.create(config.getSchema(), output);
        this.writer.setFlushOnEveryBlock(true);
        // We need to close the writer to be able to tell the name node that we created this file
        // this.writer.close();
    }
}

---