ca.etsmtl.lasi.hbasewikipedialoader.TestHBaseWikipediaLoader.java Source code

Introduction

Here is the source code for ca.etsmtl.lasi.hbasewikipedialoader.TestHBaseWikipediaLoader.java

Source

/**
 *  Copyright 2009 Jean-Daniel Cryans
 *   
 *  Licensed under the Apache License, Version 2.0 (the "License"); 
 *  you may not use this file except in compliance with the License. 
 *  You may obtain a copy of the License at
 *   
 *    http://www.apache.org/licenses/LICENSE-2.0
 *     
 *  Unless required by applicable law or agreed to in writing, software 
 *  distributed under the License is distributed on an "AS IS" BASIS, 
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 *  See the License for the specific language governing permissions and 
 *  limitations under the License. 
 */
package ca.etsmtl.lasi.hbasewikipedialoader;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;

import junit.framework.TestCase;

public class TestHBaseWikipediaLoader extends TestCase {

    /**
     * Run the loader on the sample, test if it succeeded and
     * if the number of reduced articles is the same as the number of
     * rows in the table. This test expects that HBase was started on default
     * ports on the local machine.
     */
    public void testWikipediaLoader() {
        try {
            HBaseConfiguration conf = new HBaseConfiguration();
            String[] args = new String[] { "sample/sample.xml" };
            JobConf jobConf = HBaseWikipediaLoader.createSubmittableJob(conf, args);
            RunningJob job = JobClient.runJob(jobConf);
            job.waitForCompletion();
            assertTrue(job.isSuccessful());
            HTable htable = new HTable(conf, HBaseWikipediaLoader.TABLE);
            Scan scan = new Scan();
            scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("id"));
            htable.setScannerCaching(100);
            ResultScanner scanner = htable.getScanner(scan);
            Iterator<Result> ite = scanner.iterator();
            int count = 0;
            while (ite.hasNext()) {
                Result res = ite.next();
                if (res.getRow() == null) {
                    break;
                }
                count++;
            }
            scanner.close();
            assertTrue(job.getCounters().getCounter(HBaseWikipediaLoader.Counters.MAPPED_WIKI_ARTICLES) == count);
        } catch (IOException ex) {
            ex.printStackTrace();
            fail(ex.getMessage());
        }

    }

}