List of usage examples for org.apache.hadoop.fs FSDataOutputStream FSDataOutputStream
public FSDataOutputStream(OutputStream out, FileSystem.Statistics stats)
From source file:org.apache.tajo.storage.s3.SmallBlockS3FileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path file, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { INode inode = store.retrieveINode(makeAbsolute(file)); if (inode != null) { if (overwrite) { delete(file, true);//from ww w.j a va2 s . c om } else { throw new IOException("File already exists: " + file); } } else { Path parent = file.getParent(); if (parent != null) { if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } } } return new FSDataOutputStream( new S3OutputStream(getConf(), store, makeAbsolute(file), blockSize, progress, bufferSize), statistics); }
From source file:org.xtreemfs.common.clients.hadoop.XtreemFSFileSystem.java
License:BSD License
@Override public FSDataOutputStream create(Path path, FsPermission fp, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable p) throws IOException { // block replication for the file Volume xtreemfsVolume = getVolumeFromPath(path); final String pathString = preparePath(path, xtreemfsVolume); int flags = SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber() | SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber(); if (overwrite) { flags |= SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_TRUNC.getNumber(); }//from www . j a v a2s .c o m if (Logging.isDebug()) { Logging.logMessage(Logging.LEVEL_DEBUG, this, "Creating file %s. Overwrite = %s", pathString, overwrite); } // If some of the parent directories don't exist they should be created (with default permissions for directory). if (pathString.lastIndexOf("/") != 0) { mkdirs(path.getParent()); } final FileHandle fileHandle = xtreemfsVolume.openFile(userCredentials, pathString, flags, fp.toShort()); return new FSDataOutputStream(new XtreemFSFileOutputStream(userCredentials, fileHandle, pathString, useWriteBuffer, writeBufferSize), statistics); }
From source file:org.xtreemfs.common.clients.hadoop.XtreemFSFileSystem.java
License:BSD License
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable p) throws IOException { Volume xtreemfsVolume = getVolumeFromPath(path); final String pathString = preparePath(path, xtreemfsVolume); if (Logging.isDebug()) { Logging.logMessage(Logging.LEVEL_DEBUG, this, "Append new content to file %s.", pathString); }/*from www . j a va2 s .c o m*/ // Open file. final FileHandle fileHandle = xtreemfsVolume.openFile(userCredentials, pathString, SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber()); return new FSDataOutputStream(new XtreemFSFileOutputStream(userCredentials, fileHandle, pathString, useWriteBuffer, writeBufferSize, true), statistics); }
From source file:ras.test.hadoop.fs.InMemoryFileSystem.java
License:Apache License
/** * Creates an output stream for the specified path. * /*w ww . j av a 2 s .com*/ * Does it matter is the path is absolute or relative? * * @param path * The path to the file to be created. May not be * <code>null</code>. * @param permission * The file permissions. May be <code>null</code>. If * <code>null</code>, permissions will be set to '777'. * @param overwrite * A flag indicating whether the file should be overwritten if it * already exists. if <code>false</code> and the file does exist * an {@link IOException} will be thrown. * @param bufferSize * Ignored. * @param replication * Ignored. * @param blockSize * Ignored. * @param progress * Ignored, may be <code>null</code> */ @Override public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { path = makeAbsolute(path); String pathScheme = path.toUri().getScheme(); if (pathScheme != null) { Validate.isTrue(name.getScheme().equals(pathScheme), "Wrong file system: " + pathScheme + ", expected: " + name.getScheme()); } checkParentDirWritePermission(path); Node node = getNode(path, false); if (node != null) { if (overwrite) { if (node instanceof DirectoryNode) { throw new IOException("Can't overwrite a directory with a file: " + path); } } else { throw new IOException("File already exists: " + path); } } Path parentPath = path.getParent(); mkdirs(parentPath, permission); FileNode fnode = new FileNode(path, permission); fnode.setOwner(user); setPathMapNode(getConf(), name, path, fnode); DirectoryNode dnode = (DirectoryNode) getPathMapNode(getConf(), name, parentPath); dnode.addFile(path); Statistics stats = new Statistics(workingDirectory.toUri().getScheme()); return new FSDataOutputStream(fnode.append(), stats); }
From source file:ras.test.hadoop.fs.InMemoryFileSystem.java
License:Apache License
/** * Opens an existing file for appending. * //w w w .j av a2s.c om * @param path * The path to an existing file. * @param bufferSize * Ignored. * @param progress * Ignored. */ @Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { path = makeAbsolute(path); FileNode fnode = getFileNode(path, true); checkPermission(fnode, FsAction.WRITE); Statistics stats = new Statistics(workingDirectory.toUri().getScheme()); return new FSDataOutputStream(fnode.append(), stats); }
From source file:tachyon.hadoop.AbstractTFS.java
License:Apache License
@Override public FSDataOutputStream append(Path cPath, int bufferSize, Progressable progress) throws IOException { LOG.info("append(" + cPath + ", " + bufferSize + ", " + progress + ")"); if (mStatistics != null) { mStatistics.incrementWriteOps(1); }//from w w w .ja v a2 s. c o m TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); fromHdfsToTachyon(path); long fileId = mTFS.getFileId(path); TachyonFile file = mTFS.getFile(fileId); if (file.length() > 0) { LOG.warn("This maybe an error."); } WriteType type = getWriteType(); return new FSDataOutputStream(file.getOutStream(type), mStatistics); }
From source file:tachyon.hadoop.AbstractTFS.java
License:Apache License
/** * Attempts to create a file. Overwrite will not succeed if the path exists and is a folder. * * @param cPath path to create//from w ww . j a va 2 s . c o m * @param permission permissions of the created file/folder * @param overwrite overwrite if file exists * @param bufferSize the size in bytes of the buffer to be used * @param replication under filesystem replication factor * @param blockSize block size in bytes * @param progress queryable progress * @return an FSDataOutputStream created at the indicated path of a file * @throws IOException if overwrite is not specified and the path already exists or if the path is * a folder */ @Override public FSDataOutputStream create(Path cPath, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { LOG.info("create(" + cPath + ", " + permission + ", " + overwrite + ", " + bufferSize + ", " + replication + ", " + blockSize + ", " + progress + ")"); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } boolean asyncEnabled = mTachyonConf.getBoolean(Constants.ASYNC_ENABLED); if (!asyncEnabled) { TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); if (mTFS.exist(path)) { if (overwrite && !mTFS.getFileStatus(-1, path).isFolder) { if (!mTFS.delete(path, false)) { throw new IOException("Failed to delete existing data " + cPath); } } else { throw new IOException(cPath.toString() + " already exists. Directories cannot be " + "overwritten with create."); } } long fileId = mTFS.createFile(path, blockSize); TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf()); WriteType type = getWriteType(); return new FSDataOutputStream(file.getOutStream(type), mStatistics); } if (cPath.toString().contains(FIRST_COM_PATH) && !cPath.toString().contains("SUCCESS")) { TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); mTFS.createFile(path, blockSize); String depPath = path.getPath(); depPath = depPath.substring(depPath.indexOf(FIRST_COM_PATH) + FIRST_COM_PATH.length()); depPath = depPath.substring(0, depPath.indexOf(TachyonURI.SEPARATOR)); int depId = Integer.parseInt(depPath); LOG.info("create(" + cPath + ") : " + depPath + " " + depId); depPath = path.getPath(); depPath = depPath.substring(depPath.indexOf("part-") + 5); int index = Integer.parseInt(depPath); DependencyInfo info = mTFS.getClientDependencyInfo(depId); long fileId = info.getChildren().get(index).intValue(); LOG.info("create(" + cPath + ") : " + depPath + " " + index + " " + info + " " + fileId); TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf()); return new FSDataOutputStream(file.getOutStream(WriteType.ASYNC_THROUGH), mStatistics); } if (cPath.toString().contains(RECOMPUTE_PATH) && !cPath.toString().contains("SUCCESS")) { TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); mTFS.createFile(path, blockSize); String depPath = path.getPath(); depPath = depPath.substring(depPath.indexOf(RECOMPUTE_PATH) + RECOMPUTE_PATH.length()); depPath = depPath.substring(0, depPath.indexOf(TachyonURI.SEPARATOR)); int depId = Integer.parseInt(depPath); LOG.info("create(" + cPath + ") : " + depPath + " " + depId); depPath = path.getPath(); depPath = depPath.substring(depPath.indexOf("part-") + 5); int index = Integer.parseInt(depPath); DependencyInfo info = mTFS.getClientDependencyInfo(depId); long fileId = info.getChildren().get(index).intValue(); LOG.info("create(" + cPath + ") : " + depPath + " " + index + " " + info + " " + fileId); TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf()); return new FSDataOutputStream(file.getOutStream(WriteType.ASYNC_THROUGH), mStatistics); } TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); long fileId; WriteType type = getWriteType(); if (mTFS.exist(path)) { fileId = mTFS.getFileId(path); type = WriteType.MUST_CACHE; } else { fileId = mTFS.createFile(path, blockSize); } TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf()); return new FSDataOutputStream(file.getOutStream(type), mStatistics); }