Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.cxf.jaxrs.ext.search.tika; import java.io.IOException; import java.util.Date; import java.util.HashMap; import java.util.Map; import org.apache.cxf.jaxrs.ext.search.SearchBean; import org.apache.cxf.jaxrs.ext.search.SearchConditionParser; import org.apache.cxf.jaxrs.ext.search.fiql.FiqlParser; import org.apache.cxf.jaxrs.ext.search.lucene.LuceneQueryVisitor; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Version; import org.apache.tika.parser.pdf.PDFParser; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class TikaLuceneContentExtractorTest extends Assert { private TikaLuceneContentExtractor extractor; private Directory directory; private IndexWriter writer; private SearchConditionParser<SearchBean> parser; @Before public void setUp() throws Exception { final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9); directory = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); writer = new IndexWriter(directory, config); writer.commit(); parser = new FiqlParser<SearchBean>(SearchBean.class); extractor = new TikaLuceneContentExtractor(new PDFParser()); } @Test public void testExtractedTextContentMatchesSearchCriteria() throws Exception { final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf")); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("ct==tika").length); assertEquals(1, getHits("ct==incubation").length); assertEquals(0, getHits("ct==toolsuite").length); // meta-data assertEquals(1, getHits("Author==Bertrand*").length); } @Test public void testExtractedTextContentMatchesTypesAndDateSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents").withField("modified", Date.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); // testPDF.pdf 'modified' is set to '2007-09-14T09:02:31Z' assertEquals(1, getHits("modified=gt=2007-09-14T09:02:31Z", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("modified=le=2007-09-15T09:02:31-0500", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("modified=ge=2007-09-15", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("modified==2007-09-15", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("modified==2007-09-16", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("modified=gt=2007-09-16", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("modified=lt=2007-09-15", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("modified=gt=2007-09-16T09:02:31", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("modified=lt=2007-09-01T09:02:31", documentMetadata.getFieldTypes()).length); } @Test public void testExtractedTextContentMatchesTypesAndIntegerSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents") .withField("xmpTPg:NPages", Integer.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("xmpTPg:NPages=gt=0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages==1", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages=ge=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=gt=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=lt=1", documentMetadata.getFieldTypes()).length); } @Test public void testExtractedTextContentMatchesTypesAndByteSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents") .withField("xmpTPg:NPages", Byte.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("xmpTPg:NPages=gt=0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages==1", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages=ge=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=gt=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=lt=1", documentMetadata.getFieldTypes()).length); } @Test public void testExtractedTextContentMatchesTypesAndLongSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents") .withField("xmpTPg:NPages", Long.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("xmpTPg:NPages=gt=0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages==1", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages=ge=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=gt=1", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=lt=1", documentMetadata.getFieldTypes()).length); } @Test public void testExtractedTextContentMatchesTypesAndDoubleSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents") .withField("xmpTPg:NPages", Double.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("xmpTPg:NPages=gt=0.0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages==1.0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages=ge=1.0", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=gt=1.0", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=lt=1.0", documentMetadata.getFieldTypes()).length); } @Test public void testExtractedTextContentMatchesTypesAndFloatSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata("contents") .withField("xmpTPg:NPages", Float.class); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); assertEquals(1, getHits("xmpTPg:NPages=gt=0.0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages==1.0", documentMetadata.getFieldTypes()).length); assertEquals(1, getHits("xmpTPg:NPages=ge=1.0", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=gt=1.0", documentMetadata.getFieldTypes()).length); assertEquals(0, getHits("xmpTPg:NPages=lt=1.0", documentMetadata.getFieldTypes()).length); } @Test public void testContentSourceMatchesSearchCriteria() throws Exception { final LuceneDocumentMetadata documentMetadata = new LuceneDocumentMetadata().withSource("testPDF.pdf"); final Document document = extractor.extract(getClass().getResourceAsStream("/files/testPDF.pdf"), documentMetadata); assertNotNull("Document should not be null", document); writer.addDocument(document); writer.commit(); // Should work by exact match only assertEquals(1, getHits("source==testPDF.pdf").length); assertEquals(0, getHits("source==testPDF").length); } private ScoreDoc[] getHits(final String expression) throws IOException { return getHits(expression, new HashMap<String, Class<?>>()); } private ScoreDoc[] getHits(final String expression, final Map<String, Class<?>> fieldTypes) throws IOException { IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); try { LuceneQueryVisitor<SearchBean> visitor = new LuceneQueryVisitor<SearchBean>("ct", "contents"); visitor.setPrimitiveFieldTypeMap(fieldTypes); visitor.visit(parser.parse(expression)); ScoreDoc[] hits = searcher.search(visitor.getQuery(), null, 1000).scoreDocs; assertNotNull(hits); return hits; } finally { reader.close(); } } @After public void tearDown() throws Exception { writer.close(); directory.close(); } }