com.digitalpebble.stormcrawler.protocol.selenium.NavigationFilters.java Source code

Introduction

Here is the source code for com.digitalpebble.stormcrawler.protocol.selenium.NavigationFilters.java

Source

/**
 * Licensed to DigitalPebble Ltd under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * DigitalPebble licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.digitalpebble.stormcrawler.protocol.selenium;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.slf4j.LoggerFactory;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.NullNode;

/**
 * Wrapper for the NavigationFilter defined in a JSON configuration
 */
public class NavigationFilters extends NavigationFilter {

    public static final NavigationFilters emptyNavigationFilters = new NavigationFilters();

    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(NavigationFilters.class);

    private NavigationFilter[] filters;

    private NavigationFilters() {
        filters = new NavigationFilter[0];
    }

    public ProtocolResponse filter(RemoteWebDriver driver, Metadata metadata) {
        for (NavigationFilter filter : filters) {
            ProtocolResponse response = filter.filter(driver, metadata);
            if (response != null)
                return response;
        }
        return null;
    }

    /**
     * Loads and configure the NavigationFilters based on the storm config if
     * there is one otherwise returns an emptyNavigationFilters.
     **/
    @SuppressWarnings("rawtypes")
    public static NavigationFilters fromConf(Map stormConf) {
        String configfile = ConfUtils.getString(stormConf, "navigationfilters.config.file");
        if (StringUtils.isNotBlank(configfile)) {
            try {
                return new NavigationFilters(stormConf, configfile);
            } catch (IOException e) {
                String message = "Exception caught while loading the NavigationFilters from " + configfile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return NavigationFilters.emptyNavigationFilters;
    }

    /**
     * loads the filters from a JSON configuration file
     * 
     * @throws IOException
     */
    @SuppressWarnings("rawtypes")
    public NavigationFilters(Map stormConf, String configFile) throws IOException {
        // load the JSON configFile
        // build a JSON object out of it
        JsonNode confNode = null;
        InputStream confStream = null;
        try {
            confStream = getClass().getClassLoader().getResourceAsStream(configFile);

            ObjectMapper mapper = new ObjectMapper();
            confNode = mapper.readValue(confStream, JsonNode.class);
        } catch (Exception e) {
            throw new IOException("Unable to build JSON object from file", e);
        } finally {
            if (confStream != null) {
                confStream.close();
            }
        }

        configure(stormConf, confNode);
    }

    @SuppressWarnings("rawtypes")
    @Override
    public void configure(Map stormConf, JsonNode filtersConf) {
        // initialises the filters
        List<NavigationFilter> filterLists = new ArrayList<>();

        // get the filters part
        String name = getClass().getCanonicalName();
        filtersConf = filtersConf.get(name);

        if (filtersConf == null) {
            LOG.info("No field {} in JSON config. Skipping", name);
            filters = new NavigationFilter[0];
            return;
        }

        // conf node contains a list of objects
        Iterator<JsonNode> filterIter = filtersConf.elements();
        while (filterIter.hasNext()) {
            JsonNode afilterConf = filterIter.next();
            String filterName = "<unnamed>";
            JsonNode nameNode = afilterConf.get("name");
            if (nameNode != null) {
                filterName = nameNode.textValue();
            }
            JsonNode classNode = afilterConf.get("class");
            if (classNode == null) {
                LOG.error("Filter {} doesn't specified a 'class' attribute", filterName);
                continue;
            }
            String className = classNode.textValue().trim();
            filterName += '[' + className + ']';
            // check that it is available and implements the interface
            // NavigationFilter
            try {
                Class<?> filterClass = Class.forName(className);
                boolean subClassOK = NavigationFilter.class.isAssignableFrom(filterClass);
                if (!subClassOK) {
                    LOG.error("Filter {} does not extend NavigationFilter", filterName);
                    continue;
                }
                NavigationFilter filterInstance = (NavigationFilter) filterClass.newInstance();

                JsonNode paramNode = afilterConf.get("params");
                if (paramNode != null) {
                    filterInstance.configure(stormConf, paramNode);
                } else {
                    // Pass in a nullNode if missing
                    filterInstance.configure(stormConf, NullNode.getInstance());
                }

                filterLists.add(filterInstance);
                LOG.info("Setup {}", filterName);
            } catch (Exception e) {
                LOG.error("Can't setup {}: {}", filterName, e);
                throw new RuntimeException("Can't setup " + filterName, e);
            }
        }

        filters = filterLists.toArray(new NavigationFilter[filterLists.size()]);
    }

}