com.alexholmes.hadooputils.sort.SortTest.java Source code

Introduction

Here is the source code for com.alexholmes.hadooputils.sort.SortTest.java

Source

/*
 * Copyright 2012 Alex Holmes
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alexholmes.hadooputils.sort;

import com.alexholmes.hadooputils.TestBase;
import com.alexholmes.hadooputils.test.TextIOJobBuilder;
import com.alexholmes.hadooputils.test.TextIOLocalJobBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.InputSampler;
import org.junit.Test;

import static org.junit.Assert.assertTrue;

public class SortTest extends TestBase {

    public static void run(JobConf jobConf, TextIOJobBuilder builder, int numMapTasks, int numReduceTasks,
            InputSampler.Sampler sampler) throws Exception {

        Sort sort = new Sort();

        assertTrue(sort.runJob(jobConf, // job config
                numMapTasks, // num map tasks
                numReduceTasks, // num reduce tasks
                sampler, // sampler
                null, // job compression codec
                null, // map compression codec
                false, // LZOP index
                builder.getInputPath().toUri().getPath(), // input path
                builder.getOutputPath().toUri().getPath() // outputpath
        ));

        builder.verifyResults();
    }

    public void run(TextIOJobBuilder builder) throws Exception {
        run(new JobConf(new SortConfig(builder.getFs().getConf()).getConfig()), builder, 1, 1, null);
    }

    public void run(SortConfig sortConfig, TextIOJobBuilder builder) throws Exception {
        run(new JobConf(sortConfig.getConfig()), builder, 1, 1, null);
    }

    public void run(JobConf jobConf, TextIOJobBuilder builder) throws Exception {
        run(jobConf, builder, 1, 1, null);
    }

    @Test
    public void runSimple() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foobar")
                .addInput("abcdef").addExpectedOutput("abcdef").addExpectedOutput("foobar").writeInputs();

        run(builder);
    }

    @Test
    public void runCaseSensitive() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foobar")
                .addInput("GHIdef").addExpectedOutput("GHIdef").addExpectedOutput("foobar").writeInputs();

        run(builder);
    }

    @Test
    public void runIgnoreCase() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foobar")
                .addInput("GHIdef").addExpectedOutput("foobar").addExpectedOutput("GHIdef").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setIgnoreCase(true), builder);
    }

    @Test
    public void runNonUnique() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foobar")
                .addInput("foobar").addExpectedOutput("foobar").addExpectedOutput("foobar").writeInputs();

        run(builder);
    }

    @Test
    public void runUnique() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foobar")
                .addInput("foobar").addExpectedOutput("foobar").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setUnique(true), builder);
    }

    @Test
    public void runStartScenario1() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foo bar")
                .addInput("clump").addExpectedOutput("clump").addExpectedOutput("foo bar").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(1), builder);
    }

    @Test
    public void runStartScenario2() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("abc bar")
                .addInput("clump baa").addExpectedOutput("clump baa").addExpectedOutput("abc bar").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(2), builder);
    }

    @Test
    public void runEndScenario1() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foo bar")
                .addInput("clump").addExpectedOutput("clump").addExpectedOutput("foo bar").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(1).setEndKey(1), builder);
    }

    @Test
    public void runEndScenario2() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foo bar")
                .addInput("clump hump").addExpectedOutput("clump hump").addExpectedOutput("foo bar").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(1).setEndKey(2), builder);
    }

    @Test
    public void runEndScenario3() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR).addInput("foo bar")
                .addInput("clump hump").addExpectedOutput("foo bar").addExpectedOutput("clump hump").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(2).setEndKey(2), builder);
    }

    @Test
    public void runFieldSeparator() throws Exception {

        TextIOJobBuilder builder = new TextIOLocalJobBuilder(new Configuration(), TEST_ROOT_DIR)
                .addInput("foo-hump").addInput("clump-bar").addExpectedOutput("clump-bar")
                .addExpectedOutput("foo-hump").writeInputs();

        run(new SortConfig(builder.getFs().getConf()).setStartKey(2).setEndKey(2).setFieldSeparator("-"), builder);
    }
}