org.lilyproject.hadooptestfw.HBaseProxy.java Source code

Java tutorial

Introduction

Here is the source code for org.lilyproject.hadooptestfw.HBaseProxy.java

Source

/*
 * Copyright 2010 Outerthought bvba
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.lilyproject.hadooptestfw;

import javax.management.ObjectName;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.ReflectionUtils;
import org.lilyproject.hadooptestfw.fork.HBaseTestingUtility;
import org.lilyproject.util.io.Closer;
import org.lilyproject.util.jmx.JmxLiaison;
import org.lilyproject.util.test.TestHomeUtil;

/**
 * Provides access to HBase, either by starting an embedded HBase or by connecting to a running HBase.
 *
 * <p>This is intended for usage in test cases.
 *
 * <p><b>VERY VERY IMPORTANT</b>: when connecting to an existing HBase, this class will DELETE ALL ROWS
 * FROM ALL TABLES!
 */
public class HBaseProxy {
    private Mode mode;
    private Configuration conf;
    private HBaseTestingUtility hbaseTestUtil;
    private File testHome;
    private CleanupUtil cleanupUtil;
    private boolean cleanStateOnConnect = true;
    private boolean enableMapReduce = false;
    private boolean clearData = true;
    private final Log log = LogFactory.getLog(getClass());
    private ReplicationPeerUtil mbean = new ReplicationPeerUtil();

    public enum Mode {
        EMBED, CONNECT
    }

    public static String HBASE_MODE_PROP_NAME = "lily.hbaseproxy.mode";

    public HBaseProxy() throws IOException {
        this(null);
    }

    public HBaseProxy(Mode mode) throws IOException {
        this(mode, true);
    }

    /**
     * Creates new HBaseProxy
     *
     * @param mode      either EMBED or CONNECT
     * @param clearData if true, clears the data directories upon shutdown
     * @throws IOException
     */
    public HBaseProxy(Mode mode, boolean clearData) throws IOException {
        this.clearData = clearData;

        if (mode == null) {
            String hbaseModeProp = System.getProperty(HBASE_MODE_PROP_NAME);
            if (hbaseModeProp == null || hbaseModeProp.equals("") || hbaseModeProp.equals("embed")) {
                this.mode = Mode.EMBED;
            } else if (hbaseModeProp.equals("connect")) {
                this.mode = Mode.CONNECT;
            } else {
                throw new RuntimeException("Unexpected value for " + HBASE_MODE_PROP_NAME + ": " + hbaseModeProp);
            }
        } else {
            this.mode = mode;
        }
    }

    public Mode getMode() {
        return mode;
    }

    public void setTestHome(File testHome) throws IOException {
        if (mode != Mode.EMBED) {
            throw new RuntimeException("testHome should only be set when mode is EMBED");
        }
        this.testHome = testHome;
    }

    private void initTestHome() throws IOException {
        if (testHome == null) {
            testHome = TestHomeUtil.createTestHome("lily-hbaseproxy-");
        }

        FileUtils.forceMkdir(testHome);
    }

    public boolean getCleanStateOnConnect() {
        return cleanStateOnConnect;
    }

    public void setCleanStateOnConnect(boolean cleanStateOnConnect) {
        this.cleanStateOnConnect = cleanStateOnConnect;
    }

    public boolean getEnableMapReduce() {
        return enableMapReduce;
    }

    public void setEnableMapReduce(boolean enableMapReduce) {
        this.enableMapReduce = enableMapReduce;
    }

    public HBaseTestingUtility getHBaseTestingUtility() {
        return hbaseTestUtil;
    }

    public void start() throws Exception {
        start(Collections.<String, byte[]>emptyMap());
    }

    /**
     * @param timestampReusingTables map containing table name as key and column family as value. Since HBase does
     *                               not support supporting writing data older than a deletion thombstone, these tables
     *                               will be compacted and waited for until inserting data works again.
     */
    public void start(Map<String, byte[]> timestampReusingTables) throws Exception {
        System.out.println("HBaseProxy mode: " + mode);

        conf = HBaseConfiguration.create();

        switch (mode) {
        case EMBED:
            addHBaseTestProps(conf);
            addUserProps(conf);

            initTestHome();

            System.out.println("HBaseProxy embedded mode temp dir: " + testHome.getAbsolutePath());

            hbaseTestUtil = HBaseTestingUtilityFactory.create(conf, testHome, clearData);
            hbaseTestUtil.startMiniCluster(1);
            if (enableMapReduce) {
                hbaseTestUtil.startMiniMapReduceCluster(1);
            }

            writeConfiguration(testHome, conf);

            // In the past, it happened that HMaster would not become initialized, blocking later on
            // the proper shutdown of the mini cluster. Now added this as an early warning mechanism.
            long before = System.currentTimeMillis();
            while (!hbaseTestUtil.getMiniHBaseCluster().getMaster().isInitialized()) {
                if (System.currentTimeMillis() - before > 60000) {
                    throw new RuntimeException("HMaster.isInitialized() does not become true.");
                }
                System.out.println("Waiting for HMaster to be initialized");
                Thread.sleep(500);
            }

            conf = hbaseTestUtil.getConfiguration();
            cleanupUtil = new CleanupUtil(conf, getZkConnectString());
            break;
        case CONNECT:
            conf.set("hbase.zookeeper.quorum", "localhost");
            conf.set("hbase.zookeeper.property.clientPort", "2181");
            conf.set("hbase.replication", "true");

            addUserProps(conf);

            cleanupUtil = new CleanupUtil(conf, getZkConnectString());
            if (cleanStateOnConnect) {
                cleanupUtil.cleanZooKeeper();

                Map<String, byte[]> allTimestampReusingTables = new HashMap<String, byte[]>();
                allTimestampReusingTables.putAll(cleanupUtil.getDefaultTimestampReusingTables());
                allTimestampReusingTables.putAll(timestampReusingTables);
                cleanupUtil.cleanTables(allTimestampReusingTables);

                List<String> removedPeers = cleanupUtil.cleanHBaseReplicas();
                for (String removedPeer : removedPeers) {
                    waitOnReplicationPeerStopped(removedPeer);
                }
            }

            break;
        default:
            throw new RuntimeException("Unexpected mode: " + mode);
        }

        ManagementFactory.getPlatformMBeanServer().registerMBean(mbean,
                new ObjectName("LilyHBaseProxy:name=ReplicationPeer"));
    }

    /**
     * Dumps the hadoop and hbase configuration. Useful as a reference if other applications want to use the
     * same configuration to connect with the hadoop cluster.
     *
     * @param testHome directory in which to dump the configuration (it will create a conf subdir inside)
     * @param conf     the configuration
     */
    private void writeConfiguration(File testHome, Configuration conf) throws IOException {
        final File confDir = new File(testHome, "conf");
        final boolean confDirCreated = confDir.mkdir();
        if (!confDirCreated) {
            throw new IOException("failed to create " + confDir);
        }

        // dumping everything into multiple xxx-site.xml files.. so that the expected files are definitely there
        for (String filename : Arrays.asList("core-site.xml", "mapred-site.xml")) {
            final BufferedOutputStream out = new BufferedOutputStream(
                    new FileOutputStream(new File(confDir, filename)));
            try {
                conf.writeXml(out);
            } finally {
                out.close();
            }
        }
    }

    public String getZkConnectString() {
        return conf.get("hbase.zookeeper.quorum") + ":" + conf.get("hbase.zookeeper.property.clientPort");
    }

    /**
     * Adds all system property prefixed with "lily.test.hbase." to the HBase configuration.
     */
    private void addUserProps(Configuration conf) {
        Properties sysProps = System.getProperties();
        for (Map.Entry<Object, Object> entry : sysProps.entrySet()) {
            String name = entry.getKey().toString();
            if (name.startsWith("lily.test.hbase.")) {
                String hbasePropName = name.substring("lily.test.".length());
                conf.set(hbasePropName, entry.getValue().toString());
            } else if (name.startsWith("lily.test.hbase-site.")) {
                String hbasePropName = name.substring("lily.test.hbase-site.".length());
                conf.set(hbasePropName, entry.getValue().toString());
            }
        }
    }

    protected static void addHBaseTestProps(Configuration conf) {
        // The following properties are from HBase's src/test/resources/hbase-site.xml
        conf.set("hbase.regionserver.msginterval", "1000");
        conf.set("hbase.client.pause", "5000");
        conf.set("hbase.client.retries.number", "4");
        conf.set("hbase.master.meta.thread.rescanfrequency", "10000");
        conf.set("hbase.server.thread.wakefrequency", "1000");
        conf.set("hbase.regionserver.handler.count", "5");
        conf.set("hbase.master.info.port", "-1");
        conf.set("hbase.regionserver.info.port", "-1");
        conf.set("hbase.regionserver.info.port.auto", "true");
        conf.set("hbase.master.lease.thread.wakefrequency", "3000");
        conf.set("hbase.regionserver.optionalcacheflushinterval", "1000");
        conf.set("hbase.regionserver.safemode", "false");
    }

    public void stop() throws Exception {
        if (mode == Mode.EMBED) {
            // Since HBase mini cluster shutdown has a tendency of sometimes failing (hanging waiting on master
            // to end), add a protection for this so that we do not run indefinitely. Especially important not to
            // annoy the other projects on our Hudson server.
            Thread stopHBaseThread = new Thread() {
                @Override
                public void run() {
                    try {
                        hbaseTestUtil.shutdownMiniCluster();
                        hbaseTestUtil = null;
                    } catch (Exception e) {
                        System.out.println("Error shutting down mini cluster.");
                        e.printStackTrace();
                    }
                }
            };
            stopHBaseThread.start();
            stopHBaseThread.join(60000);
            if (stopHBaseThread.isAlive()) {
                System.err.println("Unable to stop embedded mini cluster within predetermined timeout.");
                System.err.println("Dumping stack for future investigation.");
                ReflectionUtils.printThreadInfo(new PrintWriter(System.out), "Thread dump");
                System.out.println(
                        "Will now try to interrupt the mini-cluster-stop-thread and give it some more time to end.");
                stopHBaseThread.interrupt();
                stopHBaseThread.join(20000);
                throw new Exception("Failed to stop the mini cluster within the predetermined timeout.");
            }
        }

        // Close connections with HBase and HBase's ZooKeeper handles
        //HConnectionManager.deleteConnectionInfo(CONF, true);
        HConnectionManager.deleteAllConnections(true);

        // Close all HDFS connections
        FileSystem.closeAll();

        conf = null;

        if (clearData && testHome != null) {
            TestHomeUtil.cleanupTestHome(testHome);
        }

        ManagementFactory.getPlatformMBeanServer()
                .unregisterMBean(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"));
    }

    public Configuration getConf() {
        return conf;
    }

    public FileSystem getBlobFS() throws IOException, URISyntaxException {
        if (mode == Mode.EMBED) {
            return hbaseTestUtil.getDFSCluster().getFileSystem();
        } else {
            String dfsUri = System.getProperty("lily.test.dfs");

            if (dfsUri == null) {
                dfsUri = "hdfs://localhost:8020";
            }

            return FileSystem.get(new URI(dfsUri), getConf());
        }
    }

    /**
     * Cleans all data from the hbase tables.
     *
     * <p>Should only be called when lily-server is not running.
     */
    public void cleanTables() throws Exception {
        cleanupUtil.cleanTables();
    }

    /**
     * Cleans all blobs from the hdfs blobstore
     *
     * <p>Should only be called when lily-server is not running.
     */
    public void cleanBlobStore() throws Exception {
        cleanupUtil.cleanBlobStore(getBlobFS().getUri());
    }

    public void rollHLog() throws Exception {
        HBaseAdmin admin = new HBaseAdmin(conf);
        try {
            Collection<ServerName> serverNames = admin.getClusterStatus().getServers();
            if (serverNames.size() != 1) {
                throw new RuntimeException("Expected exactly one region server, but got: " + serverNames.size());
            }
            admin.rollHLogWriter(serverNames.iterator().next().getServerName());
        } finally {
            Closer.close(admin);
        }
    }

    /**
     * Wait for all outstanding waledit's that are currently in the hlog(s) to be replicated. Any new waledits
     * produced during the calling of this method won't be processed.
     *
     * <p>To avoid any timing issues, after adding a replication peer you will want to call
     * {@link #waitOnReplicationPeerReady(String)} to be sure the current logs are in the queue
     * of the new peer and that the peer's mbean is registered, otherwise this method might skip
     * that peer (usually will go so fast that this problem doesn't really exist, but just to be sure).</p>
     */
    public boolean waitOnReplication(long timeout) throws Exception {
        // Wait for the SEP to have processed all events.
        // The idea is as follows:
        //   - we want to be sure hbase replication processed all outstanding events in the hlog
        //   - therefore, roll the current hlog
        //   - if the queue of hlogs to be processed by replication contains only the current hlog (the newly
        //     rolled one), all events in the previous hlog(s) will have been processed
        //
        // It only works for one region server (which is the case for the test framework) and of course
        // assumes that new hlogs aren't being created in the meantime, but that is under all reasonable
        // circumstances the case.
        // It does ignore new edits that are happening after/during the call of this method, which is a good thing.

        // Make sure there actually is something within the hlog, otherwise it won't roll
        // This assumes the record table exits (doing the same with the .META. tables gives an exception
        // "Failed openScanner" at KeyComparator.compareWithoutRow in connect mode)
        HTable table = new HTable(conf, "-ROOT-");
        Delete delete = new Delete(Bytes.toBytes("i-am-quite-sure-this-row-does-not-exist-ha-ha-ha"));
        table.delete(delete);

        // Roll the hlog
        rollHLog();

        // Force creation of a new HLog
        delete = new Delete(Bytes.toBytes("i-am-quite-sure-this-row-does-not-exist-ha-ha-ha-2"));
        table.delete(delete);
        table.close();

        // Using JMX, query the size of the queue of hlogs to be processed for each replication source
        JmxLiaison jmxLiaison = new JmxLiaison();
        jmxLiaison.connect(mode == Mode.EMBED);
        ObjectName replicationSources = new ObjectName("hadoop:service=Replication,name=ReplicationSource for *");
        Set<ObjectName> mbeans = jmxLiaison.queryNames(replicationSources);
        long tryUntil = System.currentTimeMillis() + timeout;
        nextMBean: for (ObjectName mbean : mbeans) {
            int logQSize = Integer.MAX_VALUE;
            while (logQSize > 0 && System.currentTimeMillis() < tryUntil) {
                logQSize = (Integer) jmxLiaison.getAttribute(mbean, "sizeOfLogQueue");
                // logQSize == 0 means there is one active hlog that is polled by replication
                // and none that are queued for later processing
                // System.out.println("hlog q size is " + logQSize + " for " + mbean.toString() + " max wait left is " +
                //     (tryUntil - System.currentTimeMillis()));
                if (logQSize == 0) {
                    continue nextMBean;
                } else {
                    Thread.sleep(100);
                }
            }
            return false;
        }

        return true;
    }

    /**
     * Iteratively wait until no more sep events were handled. Useful for testing sep processors that cause new sep
     * events, which can cause even more sep events, and so on.
     *
     * See notes about {@link #waitOnReplicationPeerReady(String)} in the docs of {@link #waitOnReplication(long)}
     *
     * @param timeout
     * @return
     * @throws Exception
     */
    public boolean waitOnSepIdle(long timeout) throws Exception {
        JmxLiaison jmxLiaison = new JmxLiaison();
        jmxLiaison.connect(mode == HBaseProxy.Mode.EMBED);
        Map<String, Long> currentTimeStamp = getLastSepTimestamps(jmxLiaison);
        Map<String, Long> lastTimeStamp = null;
        int count = 0;
        long tryUntil = System.currentTimeMillis() + timeout;

        while (!currentTimeStamp.equals(lastTimeStamp)) {
            if (System.currentTimeMillis() > tryUntil)
                return false;
            log.debug("waiting for sep to idle, iteration " + count++);
            waitOnReplication(timeout);
            lastTimeStamp = currentTimeStamp;
            currentTimeStamp = getLastSepTimestamps(jmxLiaison);
        }
        return true;
    }

    /**
     * obtains a map of the replication ids to the last sep event timestamp
     */
    private Map<String, Long> getLastSepTimestamps(JmxLiaison jmxLiaison) throws Exception {
        ObjectName replicationSources = new ObjectName("hadoop:service=SEP,name=*");
        Set<ObjectName> mbeans = jmxLiaison.queryNames(replicationSources);
        Map<String, Long> result = new HashMap<String, Long>(mbeans.size());
        for (ObjectName mbean : mbeans) {
            result.put(mbean.getKeyProperty("name"), (Long) jmxLiaison.getAttribute(mbean, "lastSepTimestamp"));
        }
        return result;
    }

    /**
     * After adding a new replication peer, this waits for the replication source in the region server to be started.
     */
    public void waitOnReplicationPeerReady(String peerId) throws Exception {
        if (mode == Mode.EMBED) {
            mbean.waitOnReplicationPeerReady(peerId);
        } else {
            JmxLiaison jmxLiaison = new JmxLiaison();
            jmxLiaison.connect(false);
            jmxLiaison.invoke(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"), "waitOnReplicationPeerReady",
                    peerId);
            jmxLiaison.disconnect();
        }
    }

    /**
     * After removing a replication peer, this waits for the replication source in the region server to be stopped,
     * and will as well unregister its mbean (a workaround because this is missing in hbase at the time of this
     * writing -- hbase 0.94.3)
     */
    public void waitOnReplicationPeerStopped(String peerId) throws Exception {
        if (mode == Mode.EMBED) {
            mbean.waitOnReplicationPeerStopped(peerId);
        } else {
            JmxLiaison jmxLiaison = new JmxLiaison();
            jmxLiaison.connect(false);
            jmxLiaison.invoke(new ObjectName("LilyHBaseProxy:name=ReplicationPeer"), "waitOnReplicationPeerStopped",
                    peerId);
            jmxLiaison.disconnect();
        }
    }

}