org.apache.avro.mapred.PipesCompatibleAvroRecordReader.java Source code

Introduction

Here is the source code for org.apache.avro.mapred.PipesCompatibleAvroRecordReader.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.avro.mapred;

import java.io.IOException;

import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.avro.file.FileReader;

/** 
 * A {@code RecordReader} for Avro data files that is compatible with Hadoop
 * Pipes.
 * 
 * @param <T> the type of the input object
 */
public class PipesCompatibleAvroRecordReader<T> implements RecordReader<BytesWritable, BytesWritable> {

    private AvroRecordReader<T> avroRecordReader;
    private KeyValueGetter<T> keyValueGetter;

    private AvroWrapper<T> avroWrapper;
    private NullWritable nullWritable;

    private byte[] keyBytes;
    private byte[] valueBytes;

    /**
     * Constructs an instance of {@code PipesCompatibleAvroRecordReader}
     * 
     * @throws IOException    if constructing the AvroRecordReader fails
     */
    public PipesCompatibleAvroRecordReader(JobConf job, FileSplit split, KeyValueGetter<T> keyValueGetter)
            throws IOException {
        setClassVariables(new AvroRecordReader<T>(job, split), keyValueGetter);
    }

    protected PipesCompatibleAvroRecordReader(FileReader<T> reader, FileSplit split,
            KeyValueGetter<T> keyValueGetter) throws IOException {
        setClassVariables(new AvroRecordReader<T>(reader, split), keyValueGetter);
    }

    private void setClassVariables(AvroRecordReader<T> avroRecordReader, KeyValueGetter<T> keyValueGetter) {
        this.avroRecordReader = avroRecordReader;
        this.keyValueGetter = keyValueGetter;

        avroWrapper = new AvroWrapper<T>(null);
        nullWritable = NullWritable.get();
    }

    /**
     * Returns a key for reuse
     * 
     * @return the BytesWritable key to reuse
     */
    public BytesWritable createKey() {
        return new BytesWritable();
    }

    /**
     * Returns a value for reuse
     * 
     * @return the BytesWritable value to reuse
     */
    public BytesWritable createValue() {
        return new BytesWritable();
    }

    /**
     * If another key and value exist, updates the given key and value
     * BytesWritables and returns true. Else, returns false
     * 
     * @param key           the BytesWritable key to update with the next key,
     *                      if it exists
     * @param value         the BytesWritable value to update with the next
     *                      value, if it exists
     * @return              true if the next key/value pair exists, otherwise
     *                      false
     * @throws IOException  if the AvroRecordReader or the KeyValueGetter
     *                      return an IOException
     */
    public boolean next(BytesWritable key, BytesWritable value) throws IOException {
        if (avroRecordReader.next(avroWrapper, nullWritable)) {
            keyValueGetter.give(avroWrapper.datum());

            keyBytes = keyValueGetter.getKey();
            valueBytes = keyValueGetter.getValue();

            key.set(keyBytes, 0, keyBytes.length);
            value.set(valueBytes, 0, valueBytes.length);

            return true;
        }

        return false;
    }

    /**
     * Returns the progress of the AvroRecordReader
     * 
     * @return              the progress of the AvroRecordReader
     * @throws IOException  if AvroRecordReader throws an IOException
     */
    public float getProgress() throws IOException {
        return avroRecordReader.getProgress();
    }

    /**
     * Returns the position of the AvroRecordReader
     * 
     * @return              the progress of the AvroRecordReader
     * @throws IOException  if AvroRecordReader throws an IOException 
     */
    public long getPos() throws IOException {
        return avroRecordReader.getPos();
    }

    /**
     * Closes the AvroRecordReader
     * 
     * @throws IOException  if AvroRecordReader throws an IOException
     */
    public void close() throws IOException {
        avroRecordReader.close();
    }

}