Example usage for org.apache.hadoop.fs CreateFlag OVERWRITE

List of usage examples for org.apache.hadoop.fs CreateFlag OVERWRITE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. CreateFlag
  6. OVERWRITE

Introduction

In this page you can find the example usage for org.apache.hadoop.fs CreateFlag OVERWRITE.

Prototype

CreateFlag OVERWRITE

To view the source code for org.apache.hadoop.fs CreateFlag OVERWRITE.

Click Source Link

Document

Truncate/overwrite a file.

Usage

From source file:org.apache.apex.malhar.lib.utils.IOUtilsTest.java

License:Apache License

private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException {
    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
    File file = new File("target/IOUtilsTest/testCopyPartial/input");
    createDataFile(file, dataSize);/*from w  w w.  ja v a2  s . co  m*/

    FileContext fileContext = FileContext.getFileContext();
    DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));

    Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
    DataOutputStream outputStream = fileContext.create(output,
            EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
            Options.CreateOpts.CreateParent.createParent());

    if (offset == 0) {
        IOUtils.copyPartial(inputStream, size, outputStream);
    } else {
        IOUtils.copyPartial(inputStream, offset, size, outputStream);
    }

    outputStream.close();

    Assert.assertTrue("output exists", fileContext.util().exists(output));
    Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
    //    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
}

From source file:org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings("deprecation")
@Override//w ww .  java  2 s .co m
public FSDataOutputStream createInternal(Path f, EnumSet<CreateFlag> flag, FsPermission perm, int bufSize,
        short replication, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt,
        boolean createParent) throws IOException {
    A.notNull(f, "f");

    enterBusy();

    boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
    boolean append = flag.contains(CreateFlag.APPEND);
    boolean create = flag.contains(CreateFlag.CREATE);

    OutputStream out = null;

    try {
        IgfsPath path = convert(f);
        IgfsMode mode = modeRslvr.resolveMode(path);

        if (LOG.isDebugEnabled())
            LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path="
                    + path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');

        if (mode == PROXY) {
            FSDataOutputStream os = secondaryFs.createInternal(toSecondary(f), flag, perm, bufSize, replication,
                    blockSize, progress, checksumOpt, createParent);

            if (clientLog.isLogEnabled()) {
                long logId = IgfsLogger.nextId();

                if (append)
                    clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
                else
                    clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);

                return new FSDataOutputStream(new HadoopIgfsProxyOutputStream(os, clientLog, logId));
            } else
                return os;
        } else {
            Map<String, String> permMap = F.asMap(PROP_PERMISSION, toString(perm), PROP_PREFER_LOCAL_WRITES,
                    Boolean.toString(preferLocFileWrites));

            // Create stream and close it in the 'finally' section if any sequential operation failed.
            HadoopIgfsStreamDelegate stream;

            long logId = -1;

            if (append) {
                stream = rmtClient.append(path, create, permMap);

                if (clientLog.isLogEnabled()) {
                    logId = IgfsLogger.nextId();

                    clientLog.logAppend(logId, path, mode, bufSize);
                }

                if (LOG.isDebugEnabled())
                    LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
            } else {
                stream = rmtClient.create(path, overwrite, colocateFileWrites, replication, blockSize, permMap);

                if (clientLog.isLogEnabled()) {
                    logId = IgfsLogger.nextId();

                    clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
                }

                if (LOG.isDebugEnabled())
                    LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
            }

            assert stream != null;

            HadoopIgfsOutputStream igfsOut = new HadoopIgfsOutputStream(stream, LOG, clientLog, logId);

            bufSize = Math.max(64 * 1024, bufSize);

            out = new BufferedOutputStream(igfsOut, bufSize);

            FSDataOutputStream res = new FSDataOutputStream(out, null, 0);

            // Mark stream created successfully.
            out = null;

            return res;
        }
    } finally {
        // Close if failed during stream creation.
        if (out != null)
            U.closeQuiet(out);

        leaveBusy();
    }
}

From source file:org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings("deprecation")
@Override// ww w .  j a va2s  .  c  om
public FSDataOutputStream createInternal(Path f, EnumSet<CreateFlag> flag, FsPermission perm, int bufSize,
        short replication, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt,
        boolean createParent) throws IOException {
    A.notNull(f, "f");

    enterBusy();

    boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
    boolean append = flag.contains(CreateFlag.APPEND);
    boolean create = flag.contains(CreateFlag.CREATE);

    OutputStream out = null;

    try {
        IgfsPath path = convert(f);
        IgfsMode mode = modeRslvr.resolveMode(path);

        if (LOG.isDebugEnabled())
            LOG.debug("Opening output stream in create [thread=" + Thread.currentThread().getName() + "path="
                    + path + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ']');

        if (mode == PROXY) {
            FSDataOutputStream os = secondaryFs.createInternal(toSecondary(f), flag, perm, bufSize, replication,
                    blockSize, progress, checksumOpt, createParent);

            if (clientLog.isLogEnabled()) {
                long logId = IgfsLogger.nextId();

                if (append)
                    clientLog.logAppend(logId, path, PROXY, bufSize); // Don't have stream ID.
                else
                    clientLog.logCreate(logId, path, PROXY, overwrite, bufSize, replication, blockSize);

                return new FSDataOutputStream(new IgfsHadoopProxyOutputStream(os, clientLog, logId));
            } else
                return os;
        } else {
            Map<String, String> permMap = F.asMap(PROP_PERMISSION, toString(perm), PROP_PREFER_LOCAL_WRITES,
                    Boolean.toString(preferLocFileWrites));

            // Create stream and close it in the 'finally' section if any sequential operation failed.
            IgfsHadoopStreamDelegate stream;

            long logId = -1;

            if (append) {
                stream = rmtClient.append(path, create, permMap);

                if (clientLog.isLogEnabled()) {
                    logId = IgfsLogger.nextId();

                    clientLog.logAppend(logId, path, mode, bufSize);
                }

                if (LOG.isDebugEnabled())
                    LOG.debug("Opened output stream in append [path=" + path + ", delegate=" + stream + ']');
            } else {
                stream = rmtClient.create(path, overwrite, colocateFileWrites, replication, blockSize, permMap);

                if (clientLog.isLogEnabled()) {
                    logId = IgfsLogger.nextId();

                    clientLog.logCreate(logId, path, mode, overwrite, bufSize, replication, blockSize);
                }

                if (LOG.isDebugEnabled())
                    LOG.debug("Opened output stream in create [path=" + path + ", delegate=" + stream + ']');
            }

            assert stream != null;

            IgfsHadoopOutputStream igfsOut = new IgfsHadoopOutputStream(stream, LOG, clientLog, logId);

            bufSize = Math.max(64 * 1024, bufSize);

            out = new BufferedOutputStream(igfsOut, bufSize);

            FSDataOutputStream res = new FSDataOutputStream(out, null, 0);

            // Mark stream created successfully.
            out = null;

            return res;
        }
    } finally {
        // Close if failed during stream creation.
        if (out != null)
            U.closeQuiet(out);

        leaveBusy();
    }
}

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/** @throws Exception If failed. */
public void testCreateCheckOverwrite() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
    final Path file = new Path(dir, "someFile");

    FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
            Options.CreateOpts.perms(FsPermission.getDefault()));

    out.close();//from w  ww  .j a v  a 2s. c  om

    // Check intermediate directory permissions.
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent()).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(dir.getParent().getParent()).getPermission());

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            return fs.create(file, EnumSet.noneOf(CreateFlag.class),
                    Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, PathExistsException.class, null);

    // Overwrite should be successful.
    FSDataOutputStream out1 = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
            Options.CreateOpts.perms(FsPermission.getDefault()));

    out1.close();
}

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/**
 * Ensure that when running in multithreaded mode only one create() operation succeed.
 *
 * @throws Exception If failed.// w w  w  . j av a  2s .c o  m
 */
public void testMultithreadedCreate() throws Exception {
    Path dir = new Path(new Path(primaryFsUri), "/dir");

    fs.mkdir(dir, FsPermission.getDefault(), true);

    final Path file = new Path(dir, "file");

    fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault()))
            .close();

    final AtomicInteger cnt = new AtomicInteger();

    final Collection<Integer> errs = new GridConcurrentHashSet<>(THREAD_CNT, 1.0f, THREAD_CNT);

    multithreaded(new Runnable() {
        @Override
        public void run() {
            int idx = cnt.getAndIncrement();

            byte[] data = new byte[256];

            Arrays.fill(data, (byte) idx);

            FSDataOutputStream os = null;

            try {
                os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
                        Options.CreateOpts.perms(FsPermission.getDefault()));

                os.write(data);
            } catch (IOException ignore) {
                errs.add(idx);
            } finally {
                U.awaitQuiet(barrier);

                U.closeQuiet(os);
            }
        }
    }, THREAD_CNT);

    // Only one thread could obtain write lock on the file.
    assert errs.size() == THREAD_CNT - 1 : "Invalid errors count [expected=" + (THREAD_CNT - 1) + ", actual="
            + errs.size() + ']';

    int idx = -1;

    for (int i = 0; i < THREAD_CNT; i++) {
        if (!errs.remove(i)) {
            idx = i;

            break;
        }
    }

    byte[] expData = new byte[256];

    Arrays.fill(expData, (byte) idx);

    FSDataInputStream is = fs.open(file);

    byte[] data = new byte[256];

    is.read(data);

    is.close();

    assert Arrays.equals(expData, data);
}

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/**
 * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
 *
 * @param createBufSize Buffer size used for file creation.
 * @param writeCntsInCreate Count of times to write in file creation.
 * @param openAfterCreateBufSize Buffer size used for file opening after creation.
 * @param appendBufSize Buffer size used for file appending.
 * @param writeCntsInAppend Count of times to write in file appending.
 * @param openAfterAppendBufSize Buffer size used for file opening after appending.
 * @throws Exception If failed./*  w ww  .  j  a  v  a 2s  .  c o m*/
 */
private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
        int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
    final Path igfsHome = new Path(primaryFsUri);

    Path file = new Path(igfsHome, "/someDir/someInnerDir/someFile");

    if (createBufSize == -1)
        createBufSize = fs.getServerDefaults().getFileBufferSize();

    if (appendBufSize == -1)
        appendBufSize = fs.getServerDefaults().getFileBufferSize();

    FSDataOutputStream os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
            Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(createBufSize));

    for (int i = 0; i < writeCntsInCreate; i++)
        os.writeInt(i);

    os.close();

    FSDataInputStream is = fs.open(file, openAfterCreateBufSize);

    for (int i = 0; i < writeCntsInCreate; i++)
        assertEquals(i, is.readInt());

    is.close();

    os = fs.create(file, EnumSet.of(CreateFlag.APPEND), Options.CreateOpts.perms(FsPermission.getDefault()),
            Options.CreateOpts.bufferSize(appendBufSize));

    for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
        os.writeInt(i);

    os.close();

    is = fs.open(file, openAfterAppendBufSize);

    for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
        assertEquals(i, is.readInt());

    is.close();
}

From source file:org.apache.solr.store.hdfs.HdfsFileWriter.java

License:Apache License

public HdfsFileWriter(FileSystem fileSystem, Path path) throws IOException {
    LOG.debug("Creating writer on {}", path);
    this.path = path;

    Configuration conf = fileSystem.getConf();
    FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
    EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
    if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
        flags.add(CreateFlag.SYNC_BLOCK);
    }// w  w w.  j  a  va2  s  .  com
    outputStream = fileSystem.create(path, FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)),
            flags, fsDefaults.getFileBufferSize(), fsDefaults.getReplication(), fsDefaults.getBlockSize(),
            null);
}

From source file:org.apache.twill.filesystem.FileContextLocation.java

License:Apache License

@Override
public OutputStream getOutputStream() throws IOException {
    return fc.create(path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
            Options.CreateOpts.createParent());
}

From source file:org.apache.twill.filesystem.FileContextLocation.java

License:Apache License

@Override
public OutputStream getOutputStream(String permission) throws IOException {
    return fc.create(path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
            Options.CreateOpts.perms(new FsPermission(permission)), Options.CreateOpts.createParent());
}

From source file:org.gridgain.grid.ggfs.GridGgfsHadoop20FileSystemAbstractSelfTest.java

License:Open Source License

/**
 * Checks consistency of create --> open --> append --> open operations with different buffer sizes.
 *
 * @param createBufSize Buffer size used for file creation.
 * @param writeCntsInCreate Count of times to write in file creation.
 * @param openAfterCreateBufSize Buffer size used for file opening after creation.
 * @param appendBufSize Buffer size used for file appending.
 * @param writeCntsInAppend Count of times to write in file appending.
 * @param openAfterAppendBufSize Buffer size used for file opening after appending.
 * @throws Exception If failed.//w  w  w.  j  av a2 s.c  o m
 */
private void checkConsistency(int createBufSize, int writeCntsInCreate, int openAfterCreateBufSize,
        int appendBufSize, int writeCntsInAppend, int openAfterAppendBufSize) throws Exception {
    final Path ggfsHome = new Path(primaryFsUri);

    Path file = new Path(ggfsHome, "/someDir/someInnerDir/someFile");

    if (createBufSize == -1)
        createBufSize = fs.getServerDefaults().getFileBufferSize();

    if (appendBufSize == -1)
        appendBufSize = fs.getServerDefaults().getFileBufferSize();

    FSDataOutputStream os = fs.create(file, EnumSet.of(CreateFlag.OVERWRITE),
            Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.bufferSize(createBufSize));

    for (int i = 0; i < writeCntsInCreate; i++)
        os.writeInt(i);

    os.close();

    FSDataInputStream is = fs.open(file, openAfterCreateBufSize);

    for (int i = 0; i < writeCntsInCreate; i++)
        assertEquals(i, is.readInt());

    is.close();

    os = fs.create(file, EnumSet.of(CreateFlag.APPEND), Options.CreateOpts.perms(FsPermission.getDefault()),
            Options.CreateOpts.bufferSize(appendBufSize));

    for (int i = writeCntsInCreate; i < writeCntsInCreate + writeCntsInAppend; i++)
        os.writeInt(i);

    os.close();

    is = fs.open(file, openAfterAppendBufSize);

    for (int i = 0; i < writeCntsInCreate + writeCntsInAppend; i++)
        assertEquals(i, is.readInt());

    is.close();
}