org.apache.s4.client.Adapter.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.s4.client.Adapter.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.s4.client;

import org.apache.s4.collector.EventListener;
import org.apache.s4.collector.EventWrapper;
import org.apache.s4.dispatcher.EventDispatcher;
import org.apache.s4.listener.EventHandler;
import org.apache.s4.message.Request;
import org.apache.s4.processor.AsynchronousEventProcessor;
import org.apache.s4.util.S4Util;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Logger;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.FileSystemXmlApplicationContext;

/**
 * Adapter to connect an S4 cluster to an external client to read from and write
 * into the S4 cluster.
 */
public class Adapter {
    private static String coreHome = "../s4_core";

    // Accept events from clients and send into S4 cluster.
    private EventDispatcher dispatcher;
    private Writer eventWriter = new Writer();

    // Listen to events from S4 cluster and send to clients.
    private org.apache.s4.collector.EventListener clusterEventListener;
    private Reader eventReader = new Reader();

    /**
     * Set the dispatcher to use for sending events into S4.
     * 
     * @param dispatcher
     */
    public void setDispatcher(EventDispatcher dispatcher) {
        this.dispatcher = dispatcher;
    }

    /**
     * clusterEventListener receives events from S4 cluster. Processing of
     * events is delegated to the eventReader, which typically forwards them to
     * appropriate clients.
     * 
     * @param clusterEventListener
     */
    public void setClusterEventListener(EventListener clusterEventListener) {
        this.clusterEventListener = clusterEventListener;
        this.clusterEventListener.setEventProcessor(eventReader);
    }

    public Adapter() {
    }

    int counts[];

    private boolean init = false;

    public void init() {
        synchronized (this) {
            init = true;
        }
    }

    /**
     * Register a list of InputStubs that will send events into the S4 cluster.
     * These events will be processed by the eventWriter: i.e. the eventWriter
     * dispatches them into the S4 cluster.
     * 
     * @param stubs
     */
    public void setInputStubs(List<InputStub> stubs) {
        for (InputStub stub : stubs) {
            // register the writer as the handler for the stub's events
            stub.addHandler(eventWriter);
        }
    }

    // collections of eventReceivers (OutputStubs) to which events will be sent
    // for forwarding to clients.
    HashMap<String, List<OutputStub>> eventReceivers = new HashMap<String, List<OutputStub>>();
    List<OutputStub> eventReceiversAny = new ArrayList<OutputStub>();
    List<OutputStub> eventReceiversAll = new ArrayList<OutputStub>();

    /**
     * Register a list of OutputStubs which process events received from the S4
     * cluster, typically forwarding them to clients.
     * 
     * The {@link OutputStub#getAcceptedStreams()} is called to determine which
     * streams the stub is interested in receiving. Accordingly, three
     * collections of stubs are created.
     * 
     * <ol>
     * <li>A mapping from stream name to OutputStubs (One-to-many). Used for
     * routing.</li>
     * <li>A list of OutputStubs that accept all streams (
     * {@code stub.getAcceptedStreams() == null}. Used for routing.</li>
     * <li>A list of all OutputStubs. This is used to iterate over all the stubs
     * exactly once.</li>
     * </ol>
     * 
     * @param stubs
     *            the list of output stubs.
     */
    public void setOutputStubs(List<OutputStub> stubs) {
        eventReceiversAll.addAll(stubs);

        for (OutputStub stub : stubs) {
            // update mapping of stream names to stubs that accept events on
            // that stream.
            List<String> streams = stub.getAcceptedStreams();
            if (streams != null) {
                for (String stream : streams) {
                    List<OutputStub> stubList = eventReceivers.get(stream);
                    if (stubList == null) {
                        stubList = new ArrayList<OutputStub>();
                        eventReceivers.put(stream, stubList);
                    }

                    stubList.add(stub);
                }
            } else {
                eventReceiversAny.add(stub);
            }
        }
    }

    /**
     * Write events from input stubs into S4 cluster.
     */
    private class Writer implements EventHandler {
        // events to be dispatched into cluster
        public void processEvent(EventWrapper eventWrapper) {
            try {
                synchronized (this) {
                    if (!init) {
                        return;
                    }
                    rawEventCount++;
                    eventCount++;
                }

                // null keys => round-robin
                // empty key list => default partitioning of underlying
                // partitioner.
                List<List<String>> keys = eventWrapper.getCompoundKeyNames();

                String stream = eventWrapper.getStreamName();

                Object event = eventWrapper.getEvent();

                if (event instanceof Request)
                    decorateRequest((Request) event);

                dispatcher.dispatchEvent(stream, keys, event);

            } catch (Exception e) {
                Logger.getLogger("adapter").info("Exception adapting event", e);
            }
        }

        private volatile int eventCount = 0;
        private volatile int rawEventCount = 0;

        // set the return stream name to the adapter cluster name
        private void decorateRequest(Request r) {
            Request.RInfo rinfo = r.getRInfo();

            if (rinfo instanceof Request.ClientRInfo) {
                String cname = clusterEventListener.getRawListener().getAppName();

                ((Request.ClientRInfo) rinfo).setStream("@" + cname);
            }
        }
    }

    /**
     * Read events from S4 cluster.
     */
    private class Reader implements AsynchronousEventProcessor {
        /**
         * Queue work for processing by OutputStubs. This method simply queues
         * the events in the appropriate stubs. An event from stream K is queued
         * in OutputStub S if either S accepts all streams, or K is contained in
         * the list {@code S.getAcceptedStreams()}.
         */
        @Override
        public void queueWork(EventWrapper eventWrapper) {
            List<OutputStub> stubs = eventReceivers.get(eventWrapper.getStreamName());

            // stubs that accept any stream
            for (OutputStub stub : eventReceiversAny)
                stub.queueWork(eventWrapper);

            // stubs that receive this stream in particular
            if (stubs != null)
                for (OutputStub stub : stubs)
                    stub.queueWork(eventWrapper);
        }

        @Override
        public int getQueueSize() {
            int sz = 0;

            for (OutputStub stub : eventReceiversAll)
                sz += stub.getQueueSize();

            return sz;
        }

    }

    private static class TestDispatcher implements EventDispatcher {
        @Override
        public void dispatchEvent(String s, Object e) {
            System.out.println("Dispatching event: " + s + ":" + e);
        }

        @Override
        public void dispatchEvent(String s, List<List<String>> k, Object e) {
            System.out.println("Dispatching event: " + s + ":" + k + ":" + e);
        }
    }

    private static class TestReturnType {
        private int ra;
        private int rb;

        @SuppressWarnings("unused")
        TestReturnType() {
        }

        TestReturnType(int a, int b) {
            this.ra = a;
            this.rb = b;
        }

        public String toString() {
            return "ra=" + ra + " rb=" + rb;
        }
    }

    @SuppressWarnings("static-access")
    public static void main(String args[]) throws IOException, InterruptedException {

        Options options = new Options();

        options.addOption(OptionBuilder.withArgName("corehome").hasArg().withDescription("core home").create("c"));

        options.addOption(
                OptionBuilder.withArgName("instanceid").hasArg().withDescription("instance id").create("i"));

        options.addOption(
                OptionBuilder.withArgName("configtype").hasArg().withDescription("configuration type").create("t"));

        options.addOption(OptionBuilder.withArgName("userconfig").hasArg()
                .withDescription("user-defined legacy data adapter configuration file").create("d"));

        CommandLineParser parser = new GnuParser();
        CommandLine commandLine = null;

        try {
            commandLine = parser.parse(options, args);
        } catch (ParseException pe) {
            System.err.println(pe.getLocalizedMessage());
            System.exit(1);
        }

        int instanceId = -1;
        if (commandLine.hasOption("i")) {
            String instanceIdStr = commandLine.getOptionValue("i");
            try {
                instanceId = Integer.parseInt(instanceIdStr);
            } catch (NumberFormatException nfe) {
                System.err.println("Bad instance id: %s" + instanceIdStr);
                System.exit(1);
            }
        }

        if (commandLine.hasOption("c")) {
            coreHome = commandLine.getOptionValue("c");
        }

        String configType = "typical";
        if (commandLine.hasOption("t")) {
            configType = commandLine.getOptionValue("t");
        }

        String userConfigFilename = null;
        if (commandLine.hasOption("d")) {
            userConfigFilename = commandLine.getOptionValue("d");
        }

        File userConfigFile = new File(userConfigFilename);
        if (!userConfigFile.isFile()) {
            System.err.println("Bad user configuration file: " + userConfigFilename);
            System.exit(1);
        }

        File coreHomeFile = new File(coreHome);
        if (!coreHomeFile.isDirectory()) {
            System.err.println("Bad core home: " + coreHome);
            System.exit(1);
        }

        if (instanceId > -1) {
            System.setProperty("instanceId", "" + instanceId);
        } else {
            System.setProperty("instanceId", "" + S4Util.getPID());
        }

        String configBase = coreHome + File.separatorChar + "conf" + File.separatorChar + configType;
        String configPath = configBase + File.separatorChar + "client-adapter-conf.xml";
        File configFile = new File(configPath);
        if (!configFile.exists()) {
            System.err.printf("adapter config file %s does not exist\n", configPath);
            System.exit(1);
        }

        // load adapter config xml
        ApplicationContext coreContext;
        coreContext = new FileSystemXmlApplicationContext("file:" + configPath);
        ApplicationContext context = coreContext;

        Adapter adapter = (Adapter) context.getBean("client_adapter");

        ApplicationContext appContext = new FileSystemXmlApplicationContext(
                new String[] { "file:" + userConfigFilename }, context);

        Map<?, ?> inputStubBeanMap = appContext.getBeansOfType(InputStub.class);
        Map<?, ?> outputStubBeanMap = appContext.getBeansOfType(OutputStub.class);

        if (inputStubBeanMap.size() == 0 && outputStubBeanMap.size() == 0) {
            System.err.println("No user-defined input/output stub beans");
            System.exit(1);
        }

        ArrayList<InputStub> inputStubs = new ArrayList<InputStub>(inputStubBeanMap.size());
        ArrayList<OutputStub> outputStubs = new ArrayList<OutputStub>(outputStubBeanMap.size());

        // add all input stubs
        for (Map.Entry<?, ?> e : inputStubBeanMap.entrySet()) {
            String beanName = (String) e.getKey();
            System.out.println("Adding InputStub " + beanName);
            inputStubs.add((InputStub) e.getValue());
        }

        // add all output stubs
        for (Map.Entry<?, ?> e : outputStubBeanMap.entrySet()) {
            String beanName = (String) e.getKey();
            System.out.println("Adding OutputStub " + beanName);
            outputStubs.add((OutputStub) e.getValue());
        }

        adapter.setInputStubs(inputStubs);
        adapter.setOutputStubs(outputStubs);

    }

    public static void clientTest() throws IOException, InterruptedException {
        BasicConfigurator.configure();

        TestDispatcher disp = new TestDispatcher();

        Adapter adapter = new Adapter();
        adapter.setDispatcher(disp);

        GenericJsonClientStub stub = new GenericJsonClientStub();
        stub.setConnectionPort(2334);

        InputStub[] in = { stub };
        OutputStub[] out = { stub };
        adapter.setInputStubs(Arrays.asList(in));
        adapter.setOutputStubs(Arrays.asList(out));

        adapter.init();
        stub.init();

        while (true) {
            Thread.sleep(10000);
            TestReturnType r = new TestReturnType(100, 200);
            adapter.eventReader.queueWork(new EventWrapper("TESTSTREAM", r, null));
        }
    }
}