Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/test/org/sakai/search/index/impl/test/ClusterFSIndexStorageEclipsetest.java $ * $Id: ClusterFSIndexStorageEclipsetest.java 105078 2012-02-24 23:00:38Z ottenhoff@longsight.com $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008, 2009 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakai.search.index.impl.test; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.sql.Connection; import java.sql.Statement; import java.util.Properties; import junit.framework.TestCase; import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbcp.BasicDataSourceFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Version; import org.sakaiproject.search.index.impl.ClusterFSIndexStorage; import org.sakaiproject.search.index.impl.JDBCClusterIndexStore; import org.sakaiproject.search.index.impl.SnowballAnalyzerFactory; public class ClusterFSIndexStorageEclipsetest extends TestCase { private static final Log log = LogFactory.getLog(ClusterFSIndexStorageEclipsetest.class); ClusterFSIndexStorage cfs = null; JDBCClusterIndexStore cis = null; BasicDataSource dataSource; private String dbtype = "hsqldb"; private int docnum = 0; protected void setUp() throws Exception { super.setUp(); if ("hsqldb".equals(dbtype)) { Properties properties = new Properties(); properties.setProperty("driverClassName", "org.hsqldb.jdbcDriver"); properties.setProperty("url", "jdbc:hsqldb:mem:search-test"); properties.setProperty("maxActive", "10"); properties.setProperty("maxWait", "500"); properties.setProperty("defaultAutoCommit", "false"); properties.setProperty("defaultReadOnly", "false"); properties.setProperty("username", "sa"); properties.setProperty("password", ""); properties.setProperty("validationQuery", "select 1 from SYSTEM_USERS"); dataSource = (BasicDataSource) BasicDataSourceFactory.createDataSource(properties); cis = new JDBCClusterIndexStore(); cis.setDataSource(dataSource); Connection c = dataSource.getConnection(); Statement s = c.createStatement(); try { s.execute("CREATE TABLE search_segments " + "(" + " name_ varchar(255) not null," + " version_ BIGINT not null," + " size_ BIGINT not null" + " packet_ BINARY," + " CONSTRAINT search_segments_index UNIQUE (name_)" + "); "); } catch (Exception ex) { ex.printStackTrace(); } s.close(); c.close(); } if ("mysql".equals(dbtype)) { Properties properties = new Properties(); properties.setProperty("driverClassName", "com.mysql.jdbc.Driver"); properties.setProperty("url", "jdbc:mysql://127.0.0.1:3306/sakai22?useUnicode=true&characterEncoding=UTF-8"); properties.setProperty("maxActive", "10"); properties.setProperty("maxWait", "500"); properties.setProperty("defaultAutoCommit", "false"); properties.setProperty("defaultReadOnly", "false"); properties.setProperty("defaultTransactionIsolation", "READ_COMMITTED"); properties.setProperty("username", "sakai22"); properties.setProperty("password", "sakai22"); properties.setProperty("validationQuery", "select 1 from DUAL"); dataSource = (BasicDataSource) BasicDataSourceFactory.createDataSource(properties); cis = new JDBCClusterIndexStore(); cis.setDataSource(dataSource); Connection c = dataSource.getConnection(); Statement s = c.createStatement(); try { s.execute("CREATE TABLE search_segments" + " (" + " name_ varchar(255) not null," + " version_ BIGINT not null," + " size_ BIGINT not null," + " packet_ LONGBLOB" + " );"); s.execute("CREATE UNIQUE INDEX search_segments_index ON search_segments" + " (" + " name_" + " );"); } catch (Exception ex) { log.debug(ex); } s.close(); c.close(); } cfs = new ClusterFSIndexStorage(); cfs.setAnalyzerFactory(new SnowballAnalyzerFactory()); cfs.setClusterFS(cis); cis.setLocation("tmpindexstore"); } protected void tearDown() throws Exception { dataSource.close(); super.tearDown(); } public void disabled_testAAASaveAllSegments() throws Exception { cis.saveAllSegments(); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.getIndexReader()' */ public void testGetIndexReader() throws Exception { IndexReader ir = cfs.getIndexReader(); assertNotNull(ir); ir.numDocs(); ir.close(); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.getIndexWriter(boolean)' */ public void testGetIndexWriter() throws Exception { cfs.doPreIndexUpdate(); IndexWriter iw = cfs.getIndexWriter(false); assertNotNull(iw); Document doc = new Document(); doc.add(new Field("id", String.valueOf(System.currentTimeMillis()), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("contents", "some content about something", Field.Store.YES, Field.Index.ANALYZED)); doc.add(new Field("name", "AName", Field.Store.YES, Field.Index.NOT_ANALYZED)); iw.addDocument(doc); iw.close(); cfs.doPostIndexUpdate(); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.getIndexSearcher()' */ public void testGetIndexSearcher() throws Exception { IndexSearcher is = cfs.getIndexSearcher(false); assertNotNull(is); BooleanQuery query = new BooleanQuery(); QueryParser qp = new QueryParser(Version.LUCENE_29, "contents", cfs.getAnalyzer()); Query textQuery = qp.parse("about"); query.add(textQuery, BooleanClause.Occur.MUST); log.info("Query is " + query.toString()); TopDocs topDocs = is.search(query, 1000); log.info("Got " + topDocs.totalHits + " hits"); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.indexExists()' */ public void testIndexExists() { } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.getAnalyzer()' */ public void testGetAnalyzer() { assertNotNull(cfs.getAnalyzer()); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.doPreIndexUpdate()' */ public void testDoPreIndexUpdate() { } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.doPostIndexUpdate()' */ public void testDoPostIndexUpdate() { } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.getAnalyzerFactory()' */ public void testGetAnalyzerFactory() { assertNotNull(cfs.getAnalyzerFactory()); } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.setAnalyzerFactory(AnalyzerFactory)' */ public void testSetAnalyzerFactory() { } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.setRecoverCorruptedIndex(boolean)' */ public void testSetRecoverCorruptedIndex() { } /* * Test method for * 'org.sakaiproject.search.index.impl.ClusterFSIndexStorage.setClusterFS(ClusterFilesystem)' */ public void testSetClusterFS() { } public void testXBigTest() throws Exception { docnum = 0; cfs.doPreIndexUpdate(); IndexWriter iw = cfs.getIndexWriter(false); File f = new File("testcontent"); iw = loadDocuments(f, iw); iw.close(); cfs.doPostIndexUpdate(); } private IndexWriter reopen(IndexWriter iw) throws IOException, ParseException { log.info("Optimize==============="); iw.optimize(); iw.close(); iw = null; cfs.doPostIndexUpdate(); log.info("Query=================="); long start = System.currentTimeMillis(); IndexSearcher is = cfs.getIndexSearcher(false); log.info("----Open took " + (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); BooleanQuery query = new BooleanQuery(); QueryParser qp = new QueryParser(Version.LUCENE_29, "contents", cfs.getAnalyzer()); Query textQuery = qp.parse("sakai"); query.add(textQuery, BooleanClause.Occur.MUST); log.info("Query is " + query.toString()); TopDocs topDocs = is.search(query, 1000); log.info("Got " + topDocs.totalHits + " hits in " + (System.currentTimeMillis() - start) + " ms"); log.info("Reopen================="); cfs.doPreIndexUpdate(); iw = cfs.getIndexWriter(false); log.info("Indexing==============="); return iw; } private IndexWriter loadDocuments(File f, IndexWriter iw) throws IOException, ParseException { if (f.isDirectory()) { File[] files = f.listFiles(); for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { iw = loadDocuments(files[i], iw); } else { iw = loadDocument(files[i], iw); } } } else { iw = loadDocument(f, iw); } return iw; } private IndexWriter loadDocument(File file, IndexWriter iw) throws IOException, ParseException { docnum++; if ((docnum % 20) == 0) { iw = reopen(iw); } BufferedReader br = new BufferedReader(new FileReader(file)); StringBuilder sb = new StringBuilder(); String s = null; while ((sb.length() < 1000000) && (s = br.readLine()) != null) { sb.append(s).append("\n"); } Document doc = new Document(); doc.add(new Field("name", file.getAbsolutePath(), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("contents", sb.toString(), Field.Store.YES, Field.Index.ANALYZED)); iw.addDocument(doc); br.close(); return iw; } }