List of usage examples for org.apache.hadoop.fs FSDataOutputStream FSDataOutputStream
public FSDataOutputStream(OutputStream out, FileSystem.Statistics stats)
From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtil.java
License:Apache License
/** * Creates a new writer.//from w w w . j a va 2s . c om * @param out the drain * @param conf current configuration * @param keyClass the key type * @param valueClass the value type * @param codec the compression codec to block compression, or {@code null} to uncompressed * @return the created sequence file writer * @throws IOException if failed to create a sequence file * @throws IllegalArgumentException if some parameters were {@code null} */ public static SequenceFile.Writer openWriter(OutputStream out, Configuration conf, Class<?> keyClass, Class<?> valueClass, CompressionCodec codec) throws IOException { if (out == null) { throw new IllegalArgumentException("out must not be null"); //$NON-NLS-1$ } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ } if (keyClass == null) { throw new IllegalArgumentException("keyClass must not be null"); //$NON-NLS-1$ } if (valueClass == null) { throw new IllegalArgumentException("valueClass must not be null"); //$NON-NLS-1$ } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Creating sequence file writer for output (key={0}, value={0})", //$NON-NLS-1$ keyClass.getName(), valueClass.getName())); } FSDataOutputStream output = new FSDataOutputStream(out, null); if (codec != null) { return SequenceFile.createWriter(conf, output, keyClass, valueClass, CompressionType.BLOCK, codec); } else { return SequenceFile.createWriter(conf, output, keyClass, valueClass, CompressionType.NONE, null); } }
From source file:com.cloudera.cdk.morphline.hadoop.sequencefile.ReadSequenceFileTest.java
License:Apache License
/** * return a mapping of expected keys -> records *//*from w w w.j a va2s . c o m*/ private HashMap<String, Record> createTextSequenceFile(File file, int numRecords) throws IOException { HashMap<String, Record> map = new HashMap<String, Record>(); SequenceFile.Metadata metadata = new SequenceFile.Metadata(getMetadataForSequenceFile()); FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(file), null); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(new Configuration(), out, Text.class, Text.class, SequenceFile.CompressionType.NONE, null, metadata); for (int i = 0; i < numRecords; ++i) { Text key = new Text("key" + i); Text value = new Text("value" + i); writer.append(key, value); Record record = new Record(); record.put("key", key); record.put("value", value); map.put(key.toString(), record); } } finally { Closeables.closeQuietly(writer); } return map; }
From source file:com.cloudera.cdk.morphline.hadoop.sequencefile.ReadSequenceFileTest.java
License:Apache License
/** * return a mapping of expected keys -> records *///w w w.j a v a2 s. c o m private HashMap<String, Record> createMyWritableSequenceFile(File file, int numRecords) throws IOException { HashMap<String, Record> map = new HashMap<String, Record>(); SequenceFile.Metadata metadata = new SequenceFile.Metadata(getMetadataForSequenceFile()); FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(file), null); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(new Configuration(), out, Text.class, ParseTextMyWritableBuilder.MyWritable.class, SequenceFile.CompressionType.NONE, null, metadata); for (int i = 0; i < numRecords; ++i) { Text key = new Text("key" + i); ParseTextMyWritableBuilder.MyWritable value = new ParseTextMyWritableBuilder.MyWritable("value", i); writer.append(key, value); Record record = new Record(); record.put("key", key); record.put("value", value); map.put(key.toString(), record); } } finally { Closeables.closeQuietly(writer); } return map; }
From source file:com.cloudera.flume.handlers.seqfile.SequenceFileOutputFormat.java
License:Apache License
@Override public void format(OutputStream o, Event e) throws IOException { if (writer == null) { cachedOut = o;/*from w w w. ja va2 s . c o m*/ FSDataOutputStream fsOut; if (o instanceof FSDataOutputStream) { fsOut = (FSDataOutputStream) o; } else { fsOut = new FSDataOutputStream(o, null); } writer = SequenceFile.createWriter(FlumeConfiguration.get(), fsOut, WriteableEventKey.class, WriteableEvent.class, compressionType, codec); } if (cachedOut != o) { // different output than last time, fail here throw new IOException("OutputFormat instance can only write to the same OutputStream"); } writer.append(new WriteableEventKey(e), new WriteableEvent(e)); }
From source file:com.datatorrent.flume.storage.HDFSStorage.java
License:Open Source License
/** * This function writes the bytes to a file specified by the path * * @param path the file location/*ww w. j ava 2 s .c o m*/ * @param data the data to be written to the file * @return * @throws IOException */ private FSDataOutputStream writeData(Path path, byte[] data) throws IOException { FSDataOutputStream fsOutputStream; if (fs.getScheme().equals("file")) { // local FS does not support hflush and does not flush native stream fsOutputStream = new FSDataOutputStream( new FileOutputStream(Path.getPathWithoutSchemeAndAuthority(path).toString()), null); } else { fsOutputStream = fs.create(path); } fsOutputStream.write(data); return fsOutputStream; }
From source file:com.datatorrent.stram.FSRecoveryHandler.java
License:Apache License
@Override public DataOutputStream rotateLog() throws IOException { if (fs.exists(logBackupPath)) { // log backup is purged on snapshot/restore throw new AssertionError("Snapshot state prior to log rotation: " + logBackupPath); }//from ww w . j a v a 2s . c o m if (fs.exists(logPath)) { LOG.debug("Creating log backup {}", logBackupPath); if (!fs.rename(logPath, logBackupPath)) { throw new IOException("Failed to rotate log: " + logPath); } } LOG.info("Creating {}", logPath); final FSDataOutputStream fsOutputStream; String scheme = null; try { scheme = fs.getScheme(); } catch (UnsupportedOperationException e) { LOG.warn("{} doesn't implement getScheme() method", fs.getClass().getName()); } if ("file".equals(scheme)) { // local FS does not support hflush and does not flush native stream FSUtil.mkdirs(fs, logPath.getParent()); fsOutputStream = new FSDataOutputStream( new FileOutputStream(Path.getPathWithoutSchemeAndAuthority(logPath).toString()), null); } else { fsOutputStream = fs.create(logPath); } DataOutputStream osWrapper = new DataOutputStream(fsOutputStream) { @Override public void flush() throws IOException { super.flush(); fsOutputStream.hflush(); } @Override public void close() throws IOException { LOG.debug("Closing {}", logPath); super.close(); } }; return osWrapper; }
From source file:com.datatorrent.stram.util.FSPartFileCollection.java
License:Apache License
public void setup() throws IOException { if (basePath.startsWith("file:")) { isLocalMode = true;/*from www.j av a2s .c o m*/ localBasePath = basePath.substring(5); (new File(localBasePath)).mkdirs(); } fs = FileSystem.newInstance(new Path(basePath).toUri(), new Configuration()); Path pa = new Path(basePath, META_FILE); if (isLocalMode) { metaOs = new FSDataOutputStream(new FileOutputStream(localBasePath + "/" + META_FILE), null); } else { metaOs = fs.create(pa); } pa = new Path(basePath, INDEX_FILE); if (isLocalMode) { indexOutStr = new FSDataOutputStream(new FileOutputStream(localBasePath + "/" + INDEX_FILE), null); } else { indexOutStr = fs.create(pa); } }
From source file:com.datatorrent.stram.util.FSPartFileCollection.java
License:Apache License
private void openNewPartFile() throws IOException { hdfsFile = "part" + fileParts + ".txt"; Path path = new Path(basePath, hdfsFile); logger.debug("Opening new part file: {}", hdfsFile); if (isLocalMode) { partOutStr = new FSDataOutputStream(new FileOutputStream(localBasePath + "/" + hdfsFile), null); } else {/*from w w w. ja va 2s.co m*/ partOutStr = fs.create(path); } fileParts++; currentPartFileTimeStamp = System.currentTimeMillis(); partFileItemCount = 0; partFileBytes = 0; }
From source file:com.facebook.presto.hive.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if ((!overwrite) && exists(path)) { throw new IOException("File already exists:" + path); }/*from w ww . ja v a2 s. c om*/ createDirectories(stagingDirectory.toPath()); File tempFile = createTempFile(stagingDirectory.toPath(), "presto-s3-", ".tmp").toFile(); String key = keyFromPath(qualifiedPath(path)); return new FSDataOutputStream(new PrestoS3OutputStream(s3, uri.getHost(), key, tempFile), statistics); }
From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java
License:Apache License
@Override public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if ((!overwrite) && exists(path)) { throw new IOException("File already exists:" + path); }//from www . ja v a 2s .c o m if (!stagingDirectory.exists()) { createDirectories(stagingDirectory.toPath()); } if (!stagingDirectory.isDirectory()) { throw new IOException("Configured staging path is not a directory: " + stagingDirectory); } File tempFile = createTempFile(stagingDirectory.toPath(), "presto-s3-", ".tmp").toFile(); String key = keyFromPath(qualifiedPath(path)); return new FSDataOutputStream(new PrestoS3OutputStream(s3, transferConfig, getBucketName(uri), key, tempFile, sseEnabled, sseType, sseKmsKeyId), statistics); }