org.apache.nutch.searcher.TestDistributedSearch.java Source code

Introduction

Here is the source code for org.apache.nutch.searcher.TestDistributedSearch.java

Source

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements.  See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nutch.searcher;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.InetSocketAddress;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.Server;
import org.apache.nutch.searcher.DistributedSearch.Client;
import org.apache.nutch.util.NutchConfiguration;

public class TestDistributedSearch extends TestCase {

    private static final int DEFAULT_PORT = 60000;
    private static final String DISTRIBUTED_SEARCH_TEST_PORT = "distributed.search.test.port";

    private static final int DEFAULT_PORT1 = 60001;
    private static final String DISTRIBUTED_SEARCH_TEST_PORT1 = "distributed.search.test.port1";

    private static final int DEFAULT_PORT2 = 60002;
    private static final String DISTRIBUTED_SEARCH_TEST_PORT2 = "distributed.search.test.port2";

    Path searchdir = new Path("build/test/data/testcrawl/");

    public void testDistibutedSearch() throws IOException {

        Configuration conf = NutchConfiguration.create();

        //set up server & start it
        Server server = DistributedSearch.Server.getServer(conf, searchdir,
                conf.getInt(DISTRIBUTED_SEARCH_TEST_PORT, DEFAULT_PORT));
        server.start();

        int port = conf.getInt(DISTRIBUTED_SEARCH_TEST_PORT, DEFAULT_PORT);

        InetSocketAddress[] addresses = new InetSocketAddress[1];
        addresses[0] = new InetSocketAddress("localhost", port);

        Client c = new DistributedSearch.Client(addresses, conf);

        Query query = Query.parse("apache", conf);
        Hits hits = c.search(query, 5, null, null, false);
        c.getDetails(hits.getHit(0));
        assertTrue(hits.getTotal() > 0);

        if (server != null) {
            server.stop();
        }
    }

    public void testUpdateSegments() throws IOException {

        // Startup 2 search servers. One was already started in setup, start another 
        // one at a different port

        Configuration conf = NutchConfiguration.create();

        Server server1 = DistributedSearch.Server.getServer(conf, searchdir,
                conf.getInt(DISTRIBUTED_SEARCH_TEST_PORT1, DEFAULT_PORT1));

        Server server2 = DistributedSearch.Server.getServer(conf, searchdir,
                conf.getInt(DISTRIBUTED_SEARCH_TEST_PORT2, DEFAULT_PORT2));

        server1.start();
        server2.start();

        /* create a new file search-servers.txt
         * with 1 server at port 60000
         */
        FileSystem fs = FileSystem.get(conf);
        Path testServersPath = new Path(searchdir, "search-server.txt");
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(testServersPath, true)));
        bw.write("localhost " + DEFAULT_PORT1 + "\n");
        bw.flush();
        bw.close();

        /* 
         * Check if it found the server
         */
        Client c = new DistributedSearch.Client(testServersPath, conf);
        boolean[] liveServers = c.getLiveServer();
        assertEquals(liveServers.length, 1);

        /* Add both the servers at ports 60000 & 60005 
         * to the search-server.txt file
         */

        // give the servers a little time to wait for file modification
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        bw = new BufferedWriter(new OutputStreamWriter(fs.create(testServersPath, true)));
        bw.write("localhost " + DEFAULT_PORT1 + "\n");
        bw.write("localhost " + DEFAULT_PORT2 + "\n");
        bw.flush();
        bw.close();

        // Check if it found both the servers
        c.updateSegments();

        liveServers = c.getLiveServer();
        assertEquals(liveServers.length, 2);

        if (server1 != null) {
            server1.stop();
        }
        if (server2 != null) {
            server2.stop();
        }

        fs.delete(testServersPath, true);
    }
}