Example usage for org.apache.hadoop.fs CreateFlag SYNC_BLOCK

Introduction

In this page you can find the example usage for org.apache.hadoop.fs CreateFlag SYNC_BLOCK.

Prototype

CreateFlag SYNC_BLOCK

To view the source code for org.apache.hadoop.fs CreateFlag SYNC_BLOCK.

Click Source Link

Document

Force closed blocks to disk.

Usage

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/** Construct a new output stream for creating a file. */
private DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat, EnumSet<CreateFlag> flag,
        Progressable progress, DataChecksum checksum, String[] favoredNodes) throws IOException {
    this(dfsClient, src, progress, stat, checksum);
    this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK);

    computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);

    streamer = new DataStreamer(stat, null);
    if (favoredNodes != null && favoredNodes.length != 0) {
        streamer.setFavoredNodes(favoredNodes);
    }/*from   w w w  .java 2s . co  m*/
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/** Construct a new output stream for append. */
private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress,
        LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException {
    this(dfsClient, src, progress, stat, checksum);
    initialFileSize = stat.getLen(); // length of file when opened

    this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK);
    boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK);

    // The last partial block of the file has to be filled.
    if (!toNewBlock && lastBlock != null) {
        // indicate that we are appending to an existing block
        bytesCurBlock = lastBlock.getBlockSize();
        streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum);
    } else {/*from   w w w.jav  a  2  s.  c  o m*/
        computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);
        streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null);
    }
    this.fileEncryptionInfo = stat.getFileEncryptionInfo();
}

---

From source file:org.apache.accumulo.server.fs.VolumeManagerImpl.java

License:Apache License

@Override
public FSDataOutputStream createSyncable(Path logPath, int bufferSize, short replication, long blockSize)
        throws IOException {
    Volume v = getVolumeByPath(logPath);
    FileSystem fs = v.getFileSystem();
    blockSize = correctBlockSize(fs.getConf(), blockSize);
    bufferSize = correctBufferSize(fs.getConf(), bufferSize);
    EnumSet<CreateFlag> set = EnumSet.of(CreateFlag.SYNC_BLOCK, CreateFlag.CREATE);
    log.debug("creating " + logPath + " with CreateFlag set: " + set);
    try {// w w w .ja v a 2 s.c  om
        return fs.create(logPath, FsPermission.getDefault(), set, bufferSize, replication, blockSize, null);
    } catch (Exception ex) {
        log.debug("Exception", ex);
        return fs.create(logPath, true, bufferSize, replication, blockSize);
    }
}

---

From source file:org.apache.solr.store.hdfs.HdfsFileWriter.java

License:Apache License

public HdfsFileWriter(FileSystem fileSystem, Path path) throws IOException {
    LOG.debug("Creating writer on {}", path);
    this.path = path;

    Configuration conf = fileSystem.getConf();
    FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
    EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
    if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
        flags.add(CreateFlag.SYNC_BLOCK);
    }/*  w  w w . j a va  2s  . com*/
    outputStream = fileSystem.create(path, FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)),
            flags, fsDefaults.getFileBufferSize(), fsDefaults.getReplication(), fsDefaults.getBlockSize(),
            null);
}

---

From source file:org.elasticsearch.repositories.hdfs.HdfsBlobContainer.java

License:Apache License

@Override
public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
    if (blobExists(blobName)) {
        throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
    }//  w  w w  .j  a v  a 2 s  .  c o  m
    store.execute(new Operation<Void>() {
        @Override
        public Void run(FileContext fileContext) throws IOException {
            Path blob = new Path(path, blobName);
            // we pass CREATE, which means it fails if a blob already exists.
            // NOTE: this behavior differs from FSBlobContainer, which passes TRUNCATE_EXISTING
            // that should be fixed there, no need to bring truncation into this, give the user an error.
            EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK);
            CreateOpts[] opts = { CreateOpts.bufferSize(bufferSize) };
            try (FSDataOutputStream stream = fileContext.create(blob, flags, opts)) {
                int bytesRead;
                byte[] buffer = new byte[bufferSize];
                while ((bytesRead = inputStream.read(buffer)) != -1) {
                    stream.write(buffer, 0, bytesRead);
                    //  For safety we also hsync each write as well, because of its docs:
                    //  SYNC_BLOCK - to force closed blocks to the disk device
                    // "In addition Syncable.hsync() should be called after each write,
                    //  if true synchronous behavior is required"
                    stream.hsync();
                }
            }
            return null;
        }
    });
}

---