com.aerospike.hadoop.mapreduce.AerospikeOutputFormat.java Source code

Introduction

Here is the source code for com.aerospike.hadoop.mapreduce.AerospikeOutputFormat.java

Source

/* 
 * Copyright 2014 Aerospike, Inc.
 *
 * Portions may be licensed to Aerospike, Inc. under one or more
 * contributor license agreements.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package com.aerospike.hadoop.mapreduce;

import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.Progressable;

public abstract class AerospikeOutputFormat<KK, VV> extends OutputFormat<KK, VV>
        implements org.apache.hadoop.mapred.OutputFormat<KK, VV> {

    private static final Log log = LogFactory.getLog(AerospikeOutputFormat.class);

    public static class AerospikeOutputCommitter extends OutputCommitter {

        @Override
        public void setupJob(JobContext jobContext) throws IOException {
        }

        // compatibility check with Hadoop 0.20.2
        @Deprecated
        public void cleanupJob(JobContext jobContext) throws IOException {
        }

        @Override
        public void setupTask(TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

        @Override
        public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
            //no-op
            return false;
        }

        @Override
        public void commitTask(TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

        @Override
        public void abortTask(TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

    }

    public static class AerospikeOldAPIOutputCommitter extends org.apache.hadoop.mapred.OutputCommitter {

        @Override
        public void setupJob(org.apache.hadoop.mapred.JobContext jobContext) throws IOException {
            //no-op
        }

        @Override
        public void setupTask(org.apache.hadoop.mapred.TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

        @Override
        public boolean needsTaskCommit(org.apache.hadoop.mapred.TaskAttemptContext taskContext) throws IOException {
            //no-op
            return false;
        }

        @Override
        public void commitTask(org.apache.hadoop.mapred.TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

        @Override
        public void abortTask(org.apache.hadoop.mapred.TaskAttemptContext taskContext) throws IOException {
            //no-op
        }

        @Override
        @Deprecated
        public void cleanupJob(org.apache.hadoop.mapred.JobContext context) throws IOException {
            // no-op
            // added for compatibility with hadoop 0.20.x (used by old
            // tools, such as Cascalog)
        }
    }

    public abstract org.apache.hadoop.mapred.RecordWriter<KK, VV> getAerospikeRecordWriter(Configuration conf,
            Progressable progress);

    //
    // new API - just delegates to the Old API
    //
    @SuppressWarnings("unchecked")
    @Override
    public RecordWriter<KK, VV> getRecordWriter(TaskAttemptContext context) {
        Configuration conf = context.getConfiguration();
        return (RecordWriter<KK, VV>) getAerospikeRecordWriter(conf, context);
    }

    @Override
    public void checkOutputSpecs(JobContext context) throws IOException {
        // careful as it seems the info here saved by in the config is discarded
        Configuration cfg = context.getConfiguration();
        init(cfg);
    }

    @Override
    public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
        return new AerospikeOutputCommitter();
    }

    //
    // old API
    //
    @Override
    public org.apache.hadoop.mapred.RecordWriter<KK, VV> getRecordWriter(FileSystem ignored,
            org.apache.hadoop.mapred.JobConf job, String name, Progressable progress) {
        return getAerospikeRecordWriter(job, progress);
    }

    @Override
    public void checkOutputSpecs(FileSystem ignored, org.apache.hadoop.mapred.JobConf cfg) throws IOException {
        init(cfg);
    }

    // NB: all changes to the config objects are discarded before the
    // job is submitted if _the old MR api_ is used
    private void init(Configuration cfg) throws IOException {
        log.info(String.format("init"));
    }
}

// Local Variables:
// mode: java
// c-basic-offset: 4
// tab-width: 4
// indent-tabs-mode: nil
// End:
// vim: softtabstop=4:shiftwidth=4:expandtab