com.cloudera.science.avro.streaming.AvroAsJSONOutputFormat.java Source code

Introduction

Here is the source code for com.cloudera.science.avro.streaming.AvroAsJSONOutputFormat.java

Source

/**
 * Copyright (c) 2013, Cloudera, Inc. All Rights Reserved.
 *
 * Cloudera, Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"). You may not use this file except in
 * compliance with the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied. See the License for
 * the specific language governing permissions and limitations under the
 * License.
 */
package com.cloudera.science.avro.streaming;

import java.io.IOException;

import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileConstants;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.mapred.AvroOutputFormat;
import org.apache.avro.mapreduce.AvroJob;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.util.Progressable;

import com.cloudera.science.avro.common.JsonConverter;
import com.cloudera.science.avro.common.SchemaLoader;

/**
 * Some changes introduced in the original code by Mateusz Kobos
 */
public class AvroAsJSONOutputFormat extends FileOutputFormat<Text, Text> {
    public static final String SCHEMA_LITERAL = "output.schema.literal";
    public static final String SCHEMA_URL = "output.schema.url";
    public static final String SCHEMA_TYPE_NAME = "eu.dnetlib.iis.avro.output.class";
    public static final String READ_KEY = "output.read.key";

    private Schema schema;
    private JsonConverter converter;
    private boolean readKey = true;

    @Override
    public RecordWriter<Text, Text> getRecordWriter(FileSystem ignored, JobConf job, String name,
            Progressable progress) throws IOException {
        if (schema == null) {
            SchemaLoader loader = new SchemaLoader(job);
            this.schema = loader.load(job.get(SCHEMA_LITERAL), job.get(SCHEMA_URL), job.get(SCHEMA_TYPE_NAME));
            this.converter = new JsonConverter(schema);
            this.readKey = job.getBoolean(READ_KEY, true);
        }

        DataFileWriter<GenericRecord> writer = new DataFileWriter<GenericRecord>(
                new GenericDatumWriter<GenericRecord>(schema));
        if (getCompressOutput(job)) {
            int level = job.getInt(AvroOutputFormat.DEFLATE_LEVEL_KEY, AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
            String codecName = job.get(AvroJob.CONF_OUTPUT_CODEC,
                    org.apache.avro.file.DataFileConstants.DEFLATE_CODEC);
            CodecFactory codec = codecName.equals(DataFileConstants.DEFLATE_CODEC)
                    ? CodecFactory.deflateCodec(level)
                    : CodecFactory.fromString(codecName);
            writer.setCodec(codec);
        }
        writer.setSyncInterval(
                job.getInt(AvroOutputFormat.SYNC_INTERVAL_KEY, DataFileConstants.DEFAULT_SYNC_INTERVAL));

        Path path = FileOutputFormat.getTaskOutputPath(job, name + AvroOutputFormat.EXT);
        writer.create(schema, path.getFileSystem(job).create(path));

        return new AvroAsJSONRecordWriter(writer, converter, readKey);
    }
}