Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.summary.basic; // JDK imports import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Enumeration; import java.util.HashSet; import java.util.List; import java.util.SortedSet; import java.util.TreeSet; import java.util.Vector; // Hadoop imports import org.apache.hadoop.conf.Configuration; // Lucene imports import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; // Nutch imports import org.apache.nutch.analysis.NutchDocumentAnalyzer; import org.apache.nutch.searcher.Query; import org.apache.nutch.searcher.Summarizer; import org.apache.nutch.searcher.Summary; import org.apache.nutch.searcher.Summary.Ellipsis; import org.apache.nutch.searcher.Summary.Fragment; import org.apache.nutch.searcher.Summary.Highlight; import org.apache.nutch.util.NutchConfiguration; /** Implements hit summarization. */ public class BasicSummarizer implements Summarizer { private int sumContext = 5; private int sumLength = 20; private Analyzer analyzer = null; private Configuration conf = null; private final static Comparator ORDER_COMPARATOR = new Comparator() { public int compare(Object o1, Object o2) { return ((Excerpt) o1).getOrder() - ((Excerpt) o2).getOrder(); } }; private final static Comparator SCORE_COMPARATOR = new Comparator() { public int compare(Object o1, Object o2) { Excerpt excerpt1 = (Excerpt) o1; Excerpt excerpt2 = (Excerpt) o2; if (excerpt1 == null && excerpt2 != null) { return -1; } else if (excerpt1 != null && excerpt2 == null) { return 1; } else if (excerpt1 == null && excerpt2 == null) { return 0; } int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } }; public BasicSummarizer() { } private BasicSummarizer(Configuration conf) { setConf(conf); } /* ----------------------------- * * <implementation:Configurable> * * ----------------------------- */ public Configuration getConf() { return conf; } public void setConf(Configuration conf) { this.conf = conf; this.analyzer = new NutchDocumentAnalyzer(conf); this.sumContext = conf.getInt("searcher.summary.context", 5); this.sumLength = conf.getInt("searcher.summary.length", 20); } /* ------------------------------ * * </implementation:Configurable> * * ------------------------------ */ /* --------------------------- * * <implementation:Summarizer> * * --------------------------- */ public Summary getSummary(String text, Query query) { // Simplistic implementation. Finds the first fragments in the document // containing any query terms. // // TODO: check that phrases in the query are matched in the fragment Token[] tokens = getTokens(text); // parse text to token array if (tokens.length == 0) return new Summary(); String[] terms = query.getTerms(); HashSet highlight = new HashSet(); // put query terms in table for (int i = 0; i < terms.length; i++) highlight.add(terms[i]); // A list to store document's excerpts. // (An excerpt is a Vector full of Fragments and Highlights) List excerpts = new ArrayList(); // // Iterate through all terms in the document // int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { // // If we find a term that's in the query... // if (highlight.contains(tokens[i].termText())) { // // Start searching at a point SUM_CONTEXT terms back, // and move SUM_CONTEXT terms into the future. // int startToken = (i > sumContext) ? i - sumContext : 0; int endToken = Math.min(i + sumContext, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; // // Iterate from the start point to the finish, adding // terms all the way. The end of the passage is always // SUM_CONTEXT beyond the last query-term. // Excerpt excerpt = new Excerpt(i); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } // // Iterate through as long as we're before the end of // the document and we haven't hit the max-number-of-items // -in-a-summary. // while ((j < endToken) && (j - startToken < sumLength)) { // // Now grab the hit-element, if present // Token t = tokens[j]; if (highlight.contains(t.termText())) { excerpt.addToken(t.termText()); excerpt.add(new Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + sumContext, tokens.length); } j++; } lastExcerptPos = endToken; // // We found the series of search-term hits and added // them (with intervening text) to the excerpt. Now // we need to add the trailing edge of text. // // So if (j < tokens.length) then there is still trailing // text to add. (We haven't hit the end of the source doc.) // Add the words since the last hit-term insert. // if (j < tokens.length) { excerpt.add(new Fragment(text.substring(offset, tokens[j].endOffset()))); } // // Remember how many terms are in this excerpt // excerpt.setNumTerms(j - startToken); // // Store the excerpt for later sorting // excerpts.add(excerpt); // // Start SUM_CONTEXT places away. The next // search for relevant excerpts begins at i-SUM_CONTEXT // i = j + sumContext; } } // Sort the excerpts based on their score Collections.sort(excerpts, SCORE_COMPARATOR); // // If the target text doesn't appear, then we just // excerpt the first SUM_LENGTH words from the document. // if (excerpts.size() == 0) { Excerpt excerpt = new Excerpt(0); int excerptLen = Math.min(sumLength, tokens.length); lastExcerptPos = excerptLen; excerpt.add( new Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerpts.add(excerpt); } // // Now choose the best items from the excerpt set. // Stop when we have enought excerpts to build our Summary. // double tokenCount = 0; int numExcerpt = excerpts.size() - 1; List bestExcerpts = new ArrayList(); while (tokenCount <= sumLength && numExcerpt >= 0) { Excerpt excerpt = (Excerpt) excerpts.get(numExcerpt--); bestExcerpts.add(excerpt); tokenCount += excerpt.getNumTerms(); } // Sort the best excerpts based on their natural order Collections.sort(bestExcerpts, ORDER_COMPARATOR); // // Now build our Summary from the best the excerpts. // tokenCount = 0; numExcerpt = 0; Summary s = new Summary(); while (tokenCount <= sumLength && numExcerpt < bestExcerpts.size()) { Excerpt excerpt = (Excerpt) bestExcerpts.get(numExcerpt++); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Enumeration e = excerpt.elements(); e.hasMoreElements();) { Fragment f = (Fragment) e.nextElement(); // Don't add fragments if it takes us over the max-limit if (tokenCount + tokenFraction <= sumLength) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) s.add(new Ellipsis()); return s; } /* ---------------------------- * * </implementation:Summarizer> * * ---------------------------- */ /** Maximun number of tokens inspect in a summary . */ private static final int token_deep = 2000; /** * Class Excerpt represents a single passage found in the document, with some * appropriate regions highlit. */ class Excerpt { Vector passages = new Vector(); SortedSet tokenSet = new TreeSet(); int numTerms = 0; int order = 0; /** */ public Excerpt(int order) { this.order = order; } /** */ public void addToken(String token) { tokenSet.add(token); } /** * Return how many unique toks we have */ public int numUniqueTokens() { return tokenSet.size(); } /** * How many fragments we have. */ public int numFragments() { return passages.size(); } public void setNumTerms(int numTerms) { this.numTerms = numTerms; } public int getOrder() { return order; } public int getNumTerms() { return numTerms; } /** * Add a frag to the list. */ public void add(Fragment fragment) { passages.add(fragment); } /** * Return an Enum for all the fragments */ public Enumeration elements() { return passages.elements(); } } private Token[] getTokens(String text) { ArrayList result = new ArrayList(); TokenStream ts = analyzer.tokenStream("content", new StringReader(text)); Token token = null; while (result.size() < token_deep) { try { token = ts.next(); } catch (IOException e) { token = null; } if (token == null) { break; } result.add(token); } try { ts.close(); } catch (IOException e) { // ignore } return (Token[]) result.toArray(new Token[result.size()]); } /** * Tests Summary-generation. User inputs the name of a * text file and a query string */ public static void main(String argv[]) throws IOException { // Test arglist if (argv.length < 2) { System.out.println("Usage: java org.apache.nutch.searcher.Summarizer <textfile> <queryStr>"); return; } Configuration conf = NutchConfiguration.create(); Summarizer s = new BasicSummarizer(conf); // // Parse the args // File textFile = new File(argv[0]); StringBuffer queryBuf = new StringBuffer(); for (int i = 1; i < argv.length; i++) { queryBuf.append(argv[i]); queryBuf.append(" "); } // // Load the text file into a single string. // StringBuffer body = new StringBuffer(); BufferedReader in = new BufferedReader(new FileReader(textFile)); try { System.out.println("About to read " + textFile + " from " + in); String str = in.readLine(); while (str != null) { body.append(str); str = in.readLine(); } } finally { in.close(); } // Convert the query string into a proper Query Query query = Query.parse(queryBuf.toString(), conf); System.out.println("Summary: '" + s.getSummary(body.toString(), query) + "'"); } }