List of usage examples for org.apache.hadoop.io SequenceFile createWriter
@Deprecated public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException
From source file:co.cask.tephra.persist.HDFSTransactionLogTest.java
License:Apache License
private SequenceFile.Writer getSequenceFileWriter(Configuration configuration, FileSystem fs, long timeInMillis, boolean withMarker) throws IOException { String snapshotDir = configuration.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR); Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis); SequenceFile.Metadata metadata = new SequenceFile.Metadata(); if (withMarker) { metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY), new Text(Byte.toString(TxConstants.TransactionLog.CURRENT_VERSION))); }// w ww. ja v a2 s .co m return SequenceFile.createWriter(fs, configuration, newLog, LongWritable.class, TransactionEdit.class, SequenceFile.CompressionType.NONE, null, null, metadata); }
From source file:com.linkedin.json.JsonSequenceFileOutputFormat.java
License:Apache License
@Override public RecordWriter<Object, Object> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException { // Shamelessly copy in hadoop code to allow us to set the metadata with our schema Configuration conf = context.getConfiguration(); CompressionCodec codec = null;/* w ww .java2s . c o m*/ CompressionType compressionType = CompressionType.NONE; if (getCompressOutput(context)) { // find the kind of compression to do compressionType = SequenceFileOutputFormat.getOutputCompressionType(context); // find the right codec Class<?> codecClass = getOutputCompressorClass(context, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); } // get the path of the temporary output file Path file = getDefaultWorkFile(context, ""); FileSystem fs = file.getFileSystem(conf); final String keySchema = getSchema("output.key.schema", conf); final String valueSchema = getSchema("output.value.schema", conf); /* begin cheddar's stealing of jay's code */ SequenceFile.Metadata meta = new SequenceFile.Metadata(); meta.set(new Text("key.schema"), new Text(keySchema)); meta.set(new Text("value.schema"), new Text(valueSchema)); final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, file, context.getOutputKeyClass(), context.getOutputValueClass(), compressionType, codec, context, meta); /* end cheddar's stealing of jay's code */ final JsonTypeSerializer keySerializer = new JsonTypeSerializer(keySchema); final JsonTypeSerializer valueSerializer = new JsonTypeSerializer(valueSchema); return new RecordWriter<Object, Object>() { public void write(Object key, Object value) throws IOException { out.append(new BytesWritable(keySerializer.toBytes(key)), new BytesWritable(valueSerializer.toBytes(value))); context.progress(); } public void close(TaskAttemptContext context) throws IOException { out.close(); } }; }
From source file:org.apache.hawq.pxf.plugins.hdfs.SequenceFileAccessor.java
License:Apache License
@Override public boolean writeNextObject(OneRow onerow) throws IOException { Writable value = (Writable) onerow.getData(); Writable key = (Writable) onerow.getKey(); // init writer on first approach here, based on onerow.getData type // TODO: verify data is serializable. if (writer == null) { Class<? extends Writable> valueClass = value.getClass(); Class<? extends Writable> keyClass = (key == null) ? LongWritable.class : key.getClass(); // create writer - do not allow overwriting existing file writer = SequenceFile.createWriter(fc, conf, file, keyClass, valueClass, compressionType, codec, new SequenceFile.Metadata(), EnumSet.of(CreateFlag.CREATE)); }/*from w w w .ja v a2 s . c om*/ try { writer.append((key == null) ? defaultKey : key, value); } catch (IOException e) { LOG.error("Failed to write data to file: " + e.getMessage()); return false; } return true; }
From source file:org.apache.tephra.persist.HDFSTransactionLogTest.java
License:Apache License
private SequenceFile.Writer getSequenceFileWriter(Configuration configuration, FileSystem fs, long timeInMillis, byte versionNumber) throws IOException { String snapshotDir = configuration.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR); Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis); SequenceFile.Metadata metadata = new SequenceFile.Metadata(); if (versionNumber > 1) { metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY), new Text(Byte.toString(versionNumber))); }//from w ww . j a va 2s . c o m switch (versionNumber) { case 1: case 2: return SequenceFile.createWriter(fs, configuration, newLog, LongWritable.class, co.cask.tephra.persist.TransactionEdit.class, SequenceFile.CompressionType.NONE, null, null, metadata); default: return SequenceFile.createWriter(fs, configuration, newLog, LongWritable.class, TransactionEdit.class, SequenceFile.CompressionType.NONE, null, null, metadata); } }
From source file:org.kitesdk.examples.Main.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: java -jar zips-1.jar <zips.json> <out.sequence>"); System.exit(1);/*from www . j a v a 2 s . co m*/ } SequenceFileInputFormat in; File file = new File(args[0]); if (!file.exists() || !file.canRead()) { System.err.println("Cannot read " + file); } Schema schema = ReflectData.get().getSchema(ZipCode.class); JSONFileReader<ZipCode> reader = new JSONFileReader<ZipCode>(new FileInputStream(file), schema, ZipCode.class); reader.initialize(); FileContext context = FileContext.getLocalFSFileContext(); SequenceFile.Writer writer = SequenceFile.createWriter(context, new Configuration(), new Path(args[1]), NullWritable.class, ZipCode.class, SequenceFile.CompressionType.NONE, null, new SequenceFile.Metadata(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); for (ZipCode zip : reader) { writer.append(NullWritable.get(), zip); } writer.close(); }
From source file:voldemort.store.readonly.mr.serialization.JsonSequenceFileOutputFormat.java
License:Apache License
public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { // Shamelessly copy in hadoop code to allow us to set the metadata with // our schema // get the path of the temporary output file Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); CompressionType compressionType = CompressionType.BLOCK; // find the right codec Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); // set the schema metadata /* begin jays code */ SequenceFile.Metadata meta = new SequenceFile.Metadata(); meta.set(new Text("key.schema"), new Text(getSchema("reducer.output.key.schema", job))); meta.set(new Text("value.schema"), new Text(getSchema("reducer.output.value.schema", job))); final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, job.getOutputKeyClass(), job.getOutputValueClass(), compressionType, codec, progress, meta); /* end jays code */ return new RecordWriter<BytesWritable, BytesWritable>() { public void write(BytesWritable key, BytesWritable value) throws IOException { out.append(key, value);//w ww . jav a2 s.c o m } public void close(Reporter reporter) throws IOException { out.close(); } }; }