List of usage examples for org.apache.hadoop.fs FSDataOutputStream FSDataOutputStream
public FSDataOutputStream(OutputStream out, FileSystem.Statistics stats)
From source file:com.facebook.presto.raptor.storage.SyncingFileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path path, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if (exists(path) && !overwrite) { throw new IOException("file already exists: " + path); }//w w w. jav a2s . c o m Path parent = path.getParent(); if ((parent != null) && !mkdirs(parent)) { throw new IOException("mkdirs failed to create " + parent.toString()); } return new FSDataOutputStream( new BufferedOutputStream(new LocalFileOutputStream(pathToFile(path)), bufferSize), statistics); }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
@Override public FSDataOutputStream createObject(String objName, String contentType, Map<String, String> metadata, Statistics statistics) throws IOException { try {/*from w ww . j a v a 2 s .c om*/ String objNameWithoutBuket = objName; if (objName.startsWith(mBucket + "/")) { objNameWithoutBuket = objName.substring(mBucket.length() + 1); } if (blockUploadEnabled) { return new FSDataOutputStream(new COSBlockOutputStream(this, objNameWithoutBuket, new SemaphoredDelegatingExecutor(threadPoolExecutor, blockOutputActiveBlocks, true), partSize, blockFactory, contentType, new WriteOperationHelper(objNameWithoutBuket), metadata), null); } if (!contentType.equals(Constants.APPLICATION_DIRECTORY)) { return new FSDataOutputStream( new COSOutputStream(mBucket, objName, mClient, contentType, metadata, transfers, this), statistics); } else { final InputStream im = new InputStream() { @Override public int read() throws IOException { return -1; } }; final ObjectMetadata om = new ObjectMetadata(); om.setContentLength(0L); om.setContentType(contentType); om.setUserMetadata(metadata); // Remove the bucket name prefix from key path if (objName.startsWith(mBucket + "/")) { objName = objName.substring(mBucket.length() + 1); } /* if (!objName.endsWith("/")) { objName = objName + "/"; }*/ LOG.debug("bucket: {}, key {}", mBucket, objName); PutObjectRequest putObjectRequest = new PutObjectRequest(mBucket, objName, im, om); Upload upload = transfers.upload(putObjectRequest); upload.waitForUploadResult(); OutputStream fakeStream = new OutputStream() { @Override public void write(int b) throws IOException { } @Override public void close() throws IOException { super.close(); } }; return new FSDataOutputStream(fakeStream, statistics); } } catch (InterruptedException e) { throw new InterruptedIOException("Interrupted creating " + objName); } catch (IOException e) { LOG.error(e.getMessage()); throw e; } }
From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java
License:Open Source License
/** * Direct HTTP PUT request without JOSS package * * @param objName name of the object/*from w ww . j a va2 s . co m*/ * @param contentType content type * @return HttpURLConnection */ @Override public FSDataOutputStream createObject(String objName, String contentType, Map<String, String> metadata, Statistics statistics) throws IOException { URL url = new URL(getAccessURL() + "/" + objName); LOG.debug("PUT {}. Content-Type : {}", url.toString(), contentType); try { HttpURLConnection httpCon = (HttpURLConnection) url.openConnection(); httpCon.setDoOutput(true); httpCon.setRequestMethod("PUT"); httpCon.addRequestProperty("X-Auth-Token", getAuthToken()); httpCon.addRequestProperty("Content-Type", contentType); if (metadata != null && !metadata.isEmpty()) { for (Map.Entry<String, String> entry : metadata.entrySet()) { httpCon.addRequestProperty("X-Object-Meta-" + entry.getKey(), entry.getValue()); } } return new FSDataOutputStream(new SwiftOutputStream(httpCon), statistics); } catch (IOException e) { LOG.error(e.getMessage()); throw e; } }
From source file:com.netflix.suro.sink.localfile.FileWriterBase.java
License:Apache License
/** * Create a DataOutputStream from FSDataOutputStream. If the codec is available, * it will create compressed DataOutputStream, otherwise, it will return itself. * * @param outputStream// w ww . j ava 2 s.co m * @return * @throws java.io.IOException */ public DataOutputStream createDataOutputStream(FSDataOutputStream outputStream) throws IOException { if (codec != null) { return new FSDataOutputStream(codec.createOutputStream(outputStream), null); } else { return outputStream; } }
From source file:com.quantcast.qfs.hadoop.QFSImpl.java
License:Apache License
public FSDataOutputStream create(String path, short replication, int bufferSize, boolean overwrite, int mode, boolean append) throws IOException { return new FSDataOutputStream(createQFSOutputStream(kfsAccess, path, replication, overwrite, append, mode), statistics);// w w w .j a v a 2s. c o m }
From source file:com.quantcast.qfs.hadoop.QFSImpl.java
License:Apache License
public FSDataOutputStream create(String path, boolean overwrite, String createParams) throws IOException { if (createParams == null || createParams.length() == 0) { createParams = CREATE_PARAMS;//from www. j a v a 2 s. com } return new FSDataOutputStream(createQFSOutputStream(kfsAccess, path, overwrite, createParams), statistics); }
From source file:com.quantcast.qfs.hadoop.QFSImpl.java
License:Apache License
public FSDataOutputStream append(String path, short replication, int bufferSize) throws IOException { final boolean append = true; final boolean overwrite = false; final int mode = 0666; return new FSDataOutputStream(createQFSOutputStream(kfsAccess, path, replication, overwrite, append, mode), statistics);//www. j a va 2s. c o m }
From source file:com.quixey.hadoop.fs.oss.OSSFileSystem.java
License:Apache License
@Override @Nonnull//from w w w.j av a2 s . c o m public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, @Nullable Progressable progress) throws IOException { path = checkNotNull(path); if (exists(path) && !overwrite) { throw new FileAlreadyExistsException("File already exists: " + path); } LOG.debug("Creating new file '{}' in OSS", path); Path absolutePath = makeAbsolute(path); String key = pathToKey(absolutePath); return new FSDataOutputStream(new OSSFileOutputStream(getConf(), key, fromNullable(progress)), statistics); }
From source file:com.rapleaf.ramhdfs.RamFileSystem.java
License:Apache License
@Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progressable) throws IOException { if (!exists(f)) { throw new FileNotFoundException("File " + f + " not found."); }//from w w w . j a va2 s . co m if (isDirectory(pathToFileObject(f))) { throw new IOException("Cannot append to a diretory (=" + f + " )."); } FileObject fo = pathToFileObject(f); return new FSDataOutputStream(new BufferedOutputStream(new RamFSOutputStream(fo), bufferSize), statistics); }
From source file:com.rapleaf.ramhdfs.RamFileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if (exists(f) && !overwrite) { throw new IOException("File already exists:" + f); }/*from w w w . j a v a2 s. c om*/ Path parent = f.getParent(); if (parent != null && !mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent.toString()); } FileObject fo = pathToFileObject(f); fo.createFile(); fo.getContent().setLastModifiedTime(System.currentTimeMillis()); setPermission(f, permission); return new FSDataOutputStream(new BufferedOutputStream(new RamFSOutputStream(fo), bufferSize), statistics); }