Example usage for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter getWorkPath

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.output FileOutputCommitter getWorkPath.

Prototype

public Path getWorkPath() throws IOException 

Source Link

Document

Get the directory that the task should write results into.

Usage

From source file:com.topsoft.botspider.avro.mapreduce.output.ExtFileOutputFormat.java

License:Apache License

public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
    return new Path(committer.getWorkPath(), getUniqueFile(context, getOutputName(context), extension));
}

---

From source file:de.gesundkrank.wikipedia.hadoop.util.MapFileOutputFormat.java

License:Open Source License

@Override
public RecordWriter<WritableComparable, Writable> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);

    SequenceFile.Writer.Option keyClass = SequenceFile.Writer.keyClass(WritableComparable.class);
    SequenceFile.Writer.Option valueClass = SequenceFile.Writer.valueClass(Writable.class);
    SequenceFile.Writer.Option compressionType = SequenceFile.Writer
            .compression(SequenceFile.CompressionType.BLOCK);

    final MapFile.Writer out = new MapFile.Writer(conf, committer.getWorkPath(), keyClass, valueClass,
            compressionType);//  w w w .  ja va 2 s.  c o  m

    return new RecordWriter<WritableComparable, Writable>() {
        @Override
        public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
            out.close();
        }

        @Override
        public void write(WritableComparable key, Writable value) throws IOException, InterruptedException {
            out.append(key, value);
        }
    };
}

---

From source file:main.LicenseOutputFormat.java

@Override
public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
    // TODO Auto-generated method stub
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
    _fileName = context.getConfiguration().get(OUTPUT_FILE_NAME);
    return new Path(committer.getWorkPath(), _fileName);
}

---

From source file:org.apache.hcatalog.rcfile.RCFileMapReduceOutputFormat.java

License:Apache License

@Override
public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
        TaskAttemptContext task) throws IOException, InterruptedException {

    //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
    //TaskAttemptContext, so can't use that here
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
    Path outputPath = committer.getWorkPath();

    FileSystem fs = outputPath.getFileSystem(task.getConfiguration());

    if (!fs.exists(outputPath)) {
        fs.mkdirs(outputPath);/*from   w w w  .j  a v a2s . c o m*/
    }

    Path file = getDefaultWorkFile(task, "");

    CompressionCodec codec = null;
    if (getCompressOutput(task)) {
        Class<?> codecClass = getOutputCompressorClass(task, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, task.getConfiguration());
    }

    final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);

    return new RecordWriter<WritableComparable<?>, BytesRefArrayWritable>() {

        /* (non-Javadoc)
        * @see org.apache.hadoop.mapreduce.RecordWriter#write(java.lang.Object, java.lang.Object)
        */
        @Override
        public void write(WritableComparable<?> key, BytesRefArrayWritable value) throws IOException {
            out.append(value);
        }

        /* (non-Javadoc)
        * @see org.apache.hadoop.mapreduce.RecordWriter#close(org.apache.hadoop.mapreduce.TaskAttemptContext)
        */
        @Override
        public void close(TaskAttemptContext task) throws IOException, InterruptedException {
            out.close();
        }
    };
}

---

From source file:simsql.runtime.RecordOutputFormat.java

License:Apache License

public RecordWriter<WritableKey, WritableValue> getRecordWriter(TaskAttemptContext job)
        throws IOException, InterruptedException {

    Configuration conf = job.getConfiguration();

    // here's what we do -- if we have a map-only job and a value for
    // lastInputSplit as given to us by RecordInputFormat, then we
    // will get our part number from that file. otherwise, we'll use
    // the one we get from the job.

    // get the part from the job.
    TaskID taskId = job.getTaskAttemptID().getTaskID();
    int part = taskId.getId();
    if (RecordOutputFormat.lastInputSplit != null && taskId.getTaskType() == TaskType.MAP) {

        part = RecordOutputFormat.getPartNumber(RecordOutputFormat.lastInputSplit);
        System.out.println("MAP-ONLY JOB: USING PART NUMBER " + part + " FROM INPUT SPLIT");

        // set it back to null
        RecordOutputFormat.lastInputSplit = null;
    }//www.  j  av  a  2s  .  co m

    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(job);
    Path file = new Path(committer.getWorkPath(), RecordOutputFormat.getFileNumber(part));

    /* Path file = getDefaultWorkFile (job, ".tbl"); */
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    return new OutputFileSerializer(fileOut);
}

---

• «
• 1
• 2
• »