com.kylinolap.job.hadoop.cube.RowKeyDistributionCheckerMapper.java Source code

Introduction

Here is the source code for com.kylinolap.job.hadoop.cube.RowKeyDistributionCheckerMapper.java

Source

/*
 * Copyright 2013-2014 eBay Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.kylinolap.job.hadoop.cube;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.util.ReflectionUtils;

import com.kylinolap.common.mr.KylinMapper;

/**
 * @author ysong1
 * 
 */
public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Text, LongWritable> {

    String rowKeyStatsFilePath;
    byte[][] splitKeys;
    Map<Text, Long> resultMap;
    List<Text> keyList;

    @Override
    protected void setup(Context context) throws IOException {
        super.publishConfiguration(context.getConfiguration());

        rowKeyStatsFilePath = context.getConfiguration().get("rowKeyStatsFilePath");
        splitKeys = this.getSplits(context.getConfiguration(), new Path(rowKeyStatsFilePath));

        resultMap = new HashMap<Text, Long>();
        keyList = new ArrayList<Text>();
        for (int i = 0; i < splitKeys.length; i++) {
            Text key = new Text(splitKeys[i]);
            resultMap.put(key, 0L);
            keyList.add(new Text(splitKeys[i]));
        }
    }

    @Override
    public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
        for (Text t : keyList) {
            if (key.compareTo(t) < 0) {
                Long v = resultMap.get(t);
                long length = key.getLength() + value.getLength();
                v += length;
                resultMap.put(t, v);
                break;
            }
        }
    }

    @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
        LongWritable outputValue = new LongWritable();
        for (Entry<Text, Long> kv : resultMap.entrySet()) {
            outputValue.set(kv.getValue());
            context.write(kv.getKey(), outputValue);
        }
    }

    @SuppressWarnings("deprecation")
    public byte[][] getSplits(Configuration conf, Path path) {
        List<byte[]> rowkeyList = new ArrayList<byte[]>();
        SequenceFile.Reader reader = null;
        try {
            reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf);
            Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
            while (reader.next(key, value)) {
                byte[] tmp = ((Text) key).copyBytes();
                if (rowkeyList.contains(tmp) == false) {
                    rowkeyList.add(tmp);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            IOUtils.closeStream(reader);
        }

        byte[][] retValue = rowkeyList.toArray(new byte[rowkeyList.size()][]);

        return retValue;
    }
}