Example usage for org.apache.hadoop.mapred FileOutputFormat getTaskOutputPath

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred FileOutputFormat getTaskOutputPath.

Prototype

public static Path getTaskOutputPath(JobConf conf, String name) throws IOException

Source Link

Document

Helper function to create the task's temporary output directory and return the path to the task's output file.

Usage

From source file:org.saarus.service.hadoop.util.JsonOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = job.get("mapred.textoutputformat.separator", "\t");
    String[] headers = job.getStrings("column.headers");
    if (!isCompressed) {
        Path file = FileOutputFormat.getTaskOutputPath(job, name);
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new JsonRecordWriter<K, V>(fileOut, keyValueSeparator, headers);
    } else {//from   www  . j  a  v a2s. c om
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        // create the named codec
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job);
        // build the filename including the extension
        Path file = FileOutputFormat.getTaskOutputPath(job, name + codec.getDefaultExtension());
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new JsonRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)),
                keyValueSeparator, headers);
    }
}

From source file:redpoll.text.TermOutputFormat.java

License:Apache License

protected static RecordWriter<Text, TfArrayWritable> getTfRecordWriter(JobConf job, String name,
        Progressable progress) throws IOException {
    Path file = FileOutputFormat.getTaskOutputPath(job, name);
    FileSystem fs = file.getFileSystem(job);

    CompressionCodec codec = null;/*w w  w.j a va2  s  . c  o  m*/
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(job);
        // find the right codec
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, job);
    }

    final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class, TfArrayWritable.class,
            compressionType, codec, progress);

    return new RecordWriter<Text, TfArrayWritable>() {
        public void write(Text key, TfArrayWritable value) throws IOException {
            out.append(key, value);
        }

        public void close(Reporter reporter) throws IOException {
            out.close();
        }
    };
}

From source file:redpoll.text.TermOutputFormat.java

License:Apache License

protected static RecordWriter<Text, IntWritable> getDfRecordWriter(JobConf job, String name,
        Progressable progress) throws IOException {
    Path file = FileOutputFormat.getTaskOutputPath(job, name);
    FileSystem fs = file.getFileSystem(job);

    CompressionCodec codec = null;/*from w  w  w .  j  a  v a2  s  .  co  m*/
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(job);
        // find the right codec
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, job);
    }

    final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class, IntWritable.class,
            compressionType, codec, progress);

    return new RecordWriter<Text, IntWritable>() {
        public void write(Text key, IntWritable value) throws IOException {
            out.append(key, value);
        }

        public void close(Reporter reporter) throws IOException {
            out.close();
        }
    };
}

From source file:redpoll.text.TfIdfOutputFormat.java

License:Apache License

protected static RecordWriter<Text, WritableSparseVector> getTfIdfRecordWriter(JobConf job, String name,
        Progressable progress) throws IOException {
    Path file = FileOutputFormat.getTaskOutputPath(job, name);
    FileSystem fs = file.getFileSystem(job);

    CompressionCodec codec = null;//from ww w  . j a v a2s.com
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
        // find the kind of compression to do
        compressionType = getOutputCompressionType(job);
        // find the right codec
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, job);
    }

    final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, Text.class,
            WritableSparseVector.class, compressionType, codec, progress);

    return new RecordWriter<Text, WritableSparseVector>() {
        public void write(Text key, WritableSparseVector value) throws IOException {
            out.append(key, value);
        }

        public void close(Reporter reporter) throws IOException {
            out.close();
        }
    };
}

From source file:smile.wide.hadoop.io.LastRecordOnlyOutputFormat.java

License:Apache License

public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {
    String keyValueSeparator = job.get("mapreduce.output.textoutputformat.separator", "\t");

    Path file = FileOutputFormat.getTaskOutputPath(job, name);
    FileSystem fs = file.getFileSystem(job);
    FSDataOutputStream fileOut = fs.create(file, progress);
    return new LastRecordWriter<K, V>(fileOut, keyValueSeparator);

}

From source file:tachyon.client.keyvalue.hadoop.KeyValueOutputFormat.java

License:Apache License

@Override
public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(FileSystem ignored, JobConf conf, String name,
        Progressable progress) throws IOException {
    Path outputPath = FileOutputFormat.getTaskOutputPath(conf, name);
    KeyValueStores kvStore = KeyValueStores.Factory.create();
    try {//from w  w  w.  ja  va 2s  . c om
        KeyValueStoreWriter writer = kvStore.create(new TachyonURI(outputPath.toString()));
        return new KeyValueRecordWriter(writer, progress);
    } catch (TachyonException e) {
        throw new IOException(e);
    }
}

From source file:test.InnerTextOutputFormat.java

License:Apache License

public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = job.get("mapred.textoutputformat.separator", "\t");

    Path file = FileOutputFormat.getTaskOutputPath(job, name);
    FileSystem fs = file.getFileSystem(job);
    FSDataOutputStream fileOut = fs.create(file, progress);
    return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);

}

From source file:uk.bl.wa.hadoop.KeylessTextOutputFormat.java

License:Open Source License

@Override
public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {
    boolean isCompressed = getCompressOutput(job);
    if (!isCompressed) {
        Path file = FileOutputFormat.getTaskOutputPath(job, name);
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new LineRecordWriter<K, V>(fileOut);
    } else {/* w w w.  j  a  v a 2 s .c o  m*/
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job);
        Path file = FileOutputFormat.getTaskOutputPath(job, name + codec.getDefaultExtension());
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)));
    }
}

From source file:voldemort.store.readonly.mr.serialization.JsonSequenceFileOutputFormat.java

License:Apache License

public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(FileSystem ignored, JobConf job, String name,
        Progressable progress) throws IOException {

    // Shamelessly copy in hadoop code to allow us to set the metadata with
    // our schema

    // get the path of the temporary output file
    Path file = FileOutputFormat.getTaskOutputPath(job, name);

    FileSystem fs = file.getFileSystem(job);
    CompressionType compressionType = CompressionType.BLOCK;
    // find the right codec
    Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
    CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job);

    // set the schema metadata
    /* begin jays code */
    SequenceFile.Metadata meta = new SequenceFile.Metadata();
    meta.set(new Text("key.schema"), new Text(getSchema("reducer.output.key.schema", job)));
    meta.set(new Text("value.schema"), new Text(getSchema("reducer.output.value.schema", job)));

    final SequenceFile.Writer out = SequenceFile.createWriter(fs, job, file, job.getOutputKeyClass(),
            job.getOutputValueClass(), compressionType, codec, progress, meta);
    /* end jays code */

    return new RecordWriter<BytesWritable, BytesWritable>() {

        public void write(BytesWritable key, BytesWritable value) throws IOException {

            out.append(key, value);/*from w w w .  ja  va 2 s. co m*/
        }

        public void close(Reporter reporter) throws IOException {
            out.close();
        }
    };
}