List of usage examples for org.apache.hadoop.io SequenceFile createWriter
@Deprecated public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Progressable progress) throws IOException
From source file:org.sf.xrime.preprocessing.pajek.PajekFormat2LabeledAdjBiSetVertex.java
License:Apache License
public void toBinaryData() { try {/* w w w. ja v a 2s.c o m*/ JobConf jobConf = new JobConf(new Configuration(), PajekFormat2LabeledAdjBiSetVertex.class); Path filePath = new Path(dstPath + "/part00000"); Path path = new Path(jobConf.getWorkingDirectory(), filePath); FileSystem fs = path.getFileSystem(jobConf); CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; if (jobConf.getBoolean("mapred.output.compress", false)) { // find the kind of compression to do String val = jobConf.get("mapred.output.compression.type", CompressionType.RECORD.toString()); compressionType = CompressionType.valueOf(val); // find the right codec Class<? extends CompressionCodec> codecClass = DefaultCodec.class; String name = jobConf.get("mapred.output.compression.codec"); if (name != null) { try { codecClass = jobConf.getClassByName(name).asSubclass(CompressionCodec.class); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Compression codec " + name + " was not found.", e); } } codec = ReflectionUtils.newInstance(codecClass, jobConf); } Set<String> keySet = vertexes.keySet(); Iterator<String> iter = keySet.iterator(); LabeledAdjBiSetVertex currentAdjVertex = new LabeledAdjBiSetVertex(); SequenceFile.Writer out = SequenceFile.createWriter(fs, jobConf, path, Text.class, LabeledAdjBiSetVertex.class, compressionType, codec, null); while (iter.hasNext()) { currentAdjVertex = vertexes.get(iter.next()); out.append(new Text(currentAdjVertex.getId()), currentAdjVertex); } out.close(); } catch (IOException e) { } }
From source file:org.sf.xrime.preprocessing.pajek.PajekFormat2WeightedLabeledAdjVertex.java
License:Apache License
public void toBinaryData() { try {//from w w w . j a v a 2 s . co m JobConf jobConf = new JobConf(new Configuration(), PajekFormat2WeightedLabeledAdjVertex.class); Path filePath = new Path(dstPath + "/part00000"); Path path = new Path(jobConf.getWorkingDirectory(), filePath); FileSystem fs = path.getFileSystem(jobConf); CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; if (jobConf.getBoolean("mapred.output.compress", false)) { // find the kind of compression to do String val = jobConf.get("mapred.output.compression.type", CompressionType.RECORD.toString()); compressionType = CompressionType.valueOf(val); // find the right codec Class<? extends CompressionCodec> codecClass = DefaultCodec.class; String name = jobConf.get("mapred.output.compression.codec"); if (name != null) { try { codecClass = jobConf.getClassByName(name).asSubclass(CompressionCodec.class); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Compression codec " + name + " was not found.", e); } } codec = ReflectionUtils.newInstance(codecClass, jobConf); } Set<String> keySet = vertexes.keySet(); Iterator<String> iter = keySet.iterator(); LabeledAdjVertex currentAdjVertex = new LabeledAdjVertex(); SequenceFile.Writer out = SequenceFile.createWriter(fs, jobConf, path, Text.class, LabeledAdjVertex.class, compressionType, codec, null); while (iter.hasNext()) { currentAdjVertex = vertexes.get(iter.next()); out.append(new Text(currentAdjVertex.getId()), currentAdjVertex); } out.close(); } catch (IOException e) { } }
From source file:redpoll.text.TermOutputFormat.java
License:Apache License
protected static RecordWriter<Text, TfArrayWritable> getTfRecordWriter(JobConf job, String name, Progressable progress) throws IOException { Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); CompressionCodec codec = null;/*from w ww .j a va2 s. c o m*/ CompressionType compressionType = CompressionType.NONE; if (getCompressOutput(job)) { // find the kind of compression to do compressionType = getOutputCompressionType(job); // find the right codec Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = ReflectionUtils.newInstance(codecClass, job); } final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class, TfArrayWritable.class, compressionType, codec, progress); return new RecordWriter<Text, TfArrayWritable>() { public void write(Text key, TfArrayWritable value) throws IOException { out.append(key, value); } public void close(Reporter reporter) throws IOException { out.close(); } }; }
From source file:redpoll.text.TermOutputFormat.java
License:Apache License
protected static RecordWriter<Text, IntWritable> getDfRecordWriter(JobConf job, String name, Progressable progress) throws IOException { Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); CompressionCodec codec = null;//from ww w . ja v a 2 s .c o m CompressionType compressionType = CompressionType.NONE; if (getCompressOutput(job)) { // find the kind of compression to do compressionType = getOutputCompressionType(job); // find the right codec Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = ReflectionUtils.newInstance(codecClass, job); } final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class, IntWritable.class, compressionType, codec, progress); return new RecordWriter<Text, IntWritable>() { public void write(Text key, IntWritable value) throws IOException { out.append(key, value); } public void close(Reporter reporter) throws IOException { out.close(); } }; }
From source file:redpoll.text.TfIdfOutputFormat.java
License:Apache License
protected static RecordWriter<Text, WritableSparseVector> getTfIdfRecordWriter(JobConf job, String name, Progressable progress) throws IOException { Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); CompressionCodec codec = null;//from w ww .j ava2s . c o m CompressionType compressionType = CompressionType.NONE; if (getCompressOutput(job)) { // find the kind of compression to do compressionType = getOutputCompressionType(job); // find the right codec Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = ReflectionUtils.newInstance(codecClass, job); } final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class, WritableSparseVector.class, compressionType, codec, progress); return new RecordWriter<Text, WritableSparseVector>() { public void write(Text key, WritableSparseVector value) throws IOException { out.append(key, value); } public void close(Reporter reporter) throws IOException { out.close(); } }; }
From source file:SequenceFileWR.SWriter.java
License:Apache License
public static void main(String args[]) { JobConf conf = new JobConf(SWriter.class); Path sortWF = new Path(path, "sortFile1"); FileSystem fs = null;//from www .ja v a2 s. c om try { fs = sortWF.getFileSystem(conf); } catch (IOException e) { e.printStackTrace(); } SequenceFile.Writer out = null; try { out = SequenceFile.createWriter(fs, conf, sortWF, LongWritable.class, Text.class, CompressionType.NONE, null, null); } catch (IOException e) { e.printStackTrace(); } String refer = "Without question, many of us have mastered the neurotic art of spending" + " much of our lives worrying about variety of things -- all at once. We allow" + " past problems and future concerns to dominate your present moments, so much" + " so that we end up anxious, frustrated, depressed, and hopeless. On the flip" + " side, we also postpone our gratification, our stated priorities, and our happiness," + " often convincing ourselves that someday will be much better than today. Unfortunately," + " the same mental dynamics that tell us to look toward the future will only repeat" + " themselves so that 'someday' never actually arrives. John Lennone once said," + " Life is what is happening while we are busy making other plans.When we are busy " + " making 'other plans', our children are busy growing up, the people we love are moving" + " away and dying, our bodies are getting out of shape, and our dreams are slipping away." + " In short, we miss out on life."; String referArrays[] = refer.split("[\\s]+?"); int arrLength = referArrays.length; Random r = new Random(); for (int i = 0; i < 10000; i++) { LongWritable key = new LongWritable(r.nextLong()); Text value = new Text(referArrays[r.nextInt(arrLength)]); try { out.append(key, value); } catch (IOException e) { e.printStackTrace(); } } try { out.close(); } catch (IOException e) { e.printStackTrace(); } }