Java tutorial
/* See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * Esri Inc. licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.esri.gpt.catalog.lucene.stats; import com.esri.gpt.framework.util.Val; import java.io.IOException; import java.io.PrintWriter; import java.net.URLEncoder; import java.util.List; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; import org.apache.lucene.util.OpenBitSet; /** * Provides statistics associated with all indexed fields. * <p/> * Statistics are unavailable until collected. * <p/> * Statistics provided are: * <ul> * <li>the number of documents considered during statistics collection</li> * <li>the number of fields considered</li> * <li>the list of document frequencies per field</li> * </ul> */ class GlobalFieldStats extends Collectable { /** instance variables ====================================================== */ private FrequencyAccumulator fieldAccumulator = new FrequencyAccumulator(); /** constructors ============================================================ */ /** * Construct with a supplied term text. */ public GlobalFieldStats() { super(); } /** properties ============================================================= */ /** * Gets the number of fields considered. * @return the number of fields */ private int getFieldCount() { return this.fieldAccumulator.getFrequencies().size(); } /** * Gets the list of document frequencies per term. * <br/>Each member will be named by field and counted by document frequency. * @return the field frequencies */ private List<NamedFrequency> getFieldFrequencies() { return this.fieldAccumulator.getFrequencies(); } /** methods ================================================================= */ /** * Executes the collection of statistics. * @param request the active statistics request * @param reader the index reader * @throws IOException if an error occurs while communicating with the index */ public void collectStats(StatsRequest request, IndexReader reader) throws IOException { long t1 = System.currentTimeMillis(); TermEnum termEnum = null; TermDocs termDocs = null; try { OpenBitSet documentFilterBitSet = request.getDocumentFilterBitSet(); boolean isUnfiltered = (documentFilterBitSet == null); // return if there are no stats to collect String[] fieldNames = request.getCollectableFieldNames(reader); if (this.determineNumberOfDocsConsidered(reader, documentFilterBitSet) <= 0) { return; } else if ((fieldNames == null) || (fieldNames.length == 0)) { return; } // accumulate field frequencies per document termDocs = reader.termDocs(); for (String fieldName : fieldNames) { termEnum = reader.terms(new Term(fieldName)); OpenBitSet docsWithFieldBitSet = new OpenBitSet(reader.maxDoc()); do { Term term = termEnum.term(); if (term != null && term.field().equals(fieldName)) { termDocs.seek(term); while (termDocs.next()) { int docId = termDocs.doc(); boolean bSet = isUnfiltered || documentFilterBitSet.fastGet(docId); if (bSet) { docsWithFieldBitSet.fastSet(docId); } } } else { break; } } while (termEnum.next()); termEnum.close(); termEnum = null; if (docsWithFieldBitSet.cardinality() > 0) { this.fieldAccumulator.add(fieldName, docsWithFieldBitSet.cardinality()); } } // sort if (this.getSortByFrequency()) { this.fieldAccumulator.sortByFrequency(); } else { this.fieldAccumulator.sortByName(); } } finally { try { if (termEnum != null) termEnum.close(); } catch (Exception ef) { } try { if (termDocs != null) termDocs.close(); } catch (Exception ef) { } this.setTimeMillis(System.currentTimeMillis() - t1); } // print if (request.getResponseWriter() != null) { this.print(request); } } /** * Prints collected statistics. * @param request the active statistics request */ private void print(StatsRequest request) throws IOException { PrintWriter writer = request.getResponseWriter(); String baseUrl = request.getBaseStatsUrl(); /* writer.println("timeMillis="+this.getTimeMillis()); writer.println("numberOfDocsConsidered="+this.getNumberOfDocsConsidered()); writer.println("totalNumberOfFields="+this.getFieldFrequencies().size()); writer.println("...................."); List<NamedFrequency> docCounts = this.getFieldFrequencies(); for (NamedFrequency frequency: docCounts) { writer.println("fieldName="+frequency.getName()+", numberOfDocsWithField="+frequency.getFrequency()); } */ //writer.println("<h4>"); //writer.println("</h4>"); if (request.getResponseFormat().equalsIgnoreCase("json")) { writer.println("{"); writer.println(" \"documentsIndexed\": " + this.getNumberOfDocsConsidered() + ","); writer.println(" \"fields\": ["); List<NamedFrequency> frequencies = this.getFieldFrequencies(); int count = 0; int size = frequencies.size(); for (NamedFrequency frequency : frequencies) { count++; writer.print(" {"); writer.print("\"name\": \"" + Val.escapeStrForJson(frequency.getName()) + "\""); writer.print(", \"documents\": " + frequency.getFrequency()); if (count < size) { writer.println("},"); } else { writer.println("}"); } } writer.println(" ]"); writer.println("}"); writer.flush(); } else { // navigation writer.println("<p><a href=\"" + baseUrl + "/fields\">Fields</a>"); writer.println(" <a href=\"" + baseUrl + "/terms\">Terms</a></p>"); // summary writer.println("<p>"); writer.println("<strong>Indexed fields:</strong> " + this.getFieldFrequencies().size()); writer.println("<br/><strong>Indexed documents:</strong> " + this.getNumberOfDocsConsidered()); writer.println("</p>"); // statistics table writer.println("<table border='1'>"); writer.println("<thead><tr><th>Field</th><th>Documents</th></tr></thead>"); writer.println("<tbody>"); List<NamedFrequency> frequencies = this.getFieldFrequencies(); for (NamedFrequency frequency : frequencies) { writer.print("<tr>"); writer.print("<td>"); String href = baseUrl + "/fields?field=" + URLEncoder.encode(frequency.getName(), "UTF-8"); writer.print("<a href=\"" + href + "\">"); writer.print(Val.escapeXmlForBrowser(frequency.getName())); writer.print("</a></td>"); writer.print("<td>"); writer.print(frequency.getFrequency()); writer.print("</td>"); writer.println("</tr>"); writer.flush(); } writer.println("</tbody>"); writer.println("</table>"); writer.flush(); } } }