Java tutorial
/* LanguageTool, a natural language style checker * Copyright (C) 2005 Daniel Naber (http://www.danielnaber.de) * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 * USA */ package org.languagetool.dev.index; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; import org.apache.lucene.sandbox.queries.regex.JavaUtilRegexCapabilities; import org.apache.lucene.sandbox.queries.regex.RegexQuery; import org.apache.lucene.search.*; import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.languagetool.AnalyzedToken; import org.languagetool.Language; import org.languagetool.rules.patterns.Element; import org.languagetool.rules.patterns.PatternRule; import org.languagetool.synthesis.Synthesizer; import java.io.IOException; import java.util.HashSet; import java.util.Set; import static org.languagetool.dev.index.LanguageToolFilter.LEMMA_PREFIX; import static org.languagetool.dev.index.LanguageToolFilter.POS_PREFIX; /** * A factory class for building a Lucene Query from a PatternRule. The query * requires an index where each document contains only one sentence. It returns * potential matches, i.e. LanguageTool still needs to run over the matches * to make sure there is indeed an error. * * @author Tao Lin * @author Daniel Naber */ public class PatternRuleQueryBuilder { public static final String FIELD_NAME = "field"; public static final String SOURCE_FIELD_NAME = "source"; public static final String FIELD_NAME_LOWERCASE = "fieldLowercase"; private final Language language; public PatternRuleQueryBuilder(Language language) { this.language = language; } /** * Iterate over all elements, ignore those not supported, add the other ones to a BooleanQuery. * @throws UnsupportedPatternRuleException if no query could be created for the rule */ public Query buildRelaxedQuery(PatternRule rule) throws UnsupportedPatternRuleException { final BooleanQuery booleanQuery = new BooleanQuery(); for (Element element : rule.getElements()) { try { final BooleanClause clause = makeQuery(element); booleanQuery.add(clause); } catch (UnsupportedPatternRuleException e) { //System.out.println("Ignoring because it's not supported: " + element + ": " + e); // cannot handle - okay to ignore, as we may return too broad matches } catch (Exception e) { throw new RuntimeException("Could not create query for rule " + rule.getId(), e); } } if (booleanQuery.clauses().size() == 0) { throw new UnsupportedPatternRuleException( "No items found in rule that can be used to build a search query: " + rule); } return booleanQuery; } private BooleanClause makeQuery(Element element) throws UnsupportedPatternRuleException { checkUnsupportedElement(element); final String termStr = element.getString(); final String pos = element.getPOStag(); final BooleanClause termQuery = getTermQueryOrNull(element, termStr); final BooleanClause posQuery = getPosQueryOrNull(element, pos); if (termQuery != null && posQuery != null) { // if both term and POS are set, we create a query where both are at the same position if (mustOccur(termQuery) && mustOccur(posQuery)) { final SpanQuery spanQueryForTerm = asSpanQuery(termQuery); final SpanQuery spanQueryForPos = asSpanQuery(posQuery); final SpanQuery[] spanClauses = { spanQueryForTerm, spanQueryForPos }; return new BooleanClause(new SpanNearQuery(spanClauses, 0, false), BooleanClause.Occur.MUST); } else { // should not happen, we always use Occur.MUST: throw new UnsupportedPatternRuleException("Term/POS combination not supported yet: " + element); } } else if (termQuery != null) { return termQuery; } else if (posQuery != null) { return posQuery; } throw new UnsupportedPatternRuleException("Neither POS tag nor term set for element: " + element); } private SpanQuery asSpanQuery(BooleanClause query) { if (query.getQuery() instanceof MultiTermQuery) { return new SpanMultiTermQueryWrapper<>((MultiTermQuery) query.getQuery()); } else { final Set<Term> terms = new HashSet<>(); query.getQuery().extractTerms(terms); if (terms.size() != 1) { throw new RuntimeException("Expected term set of size 1: " + terms); } return new SpanTermQuery(terms.iterator().next()); } } private boolean mustOccur(BooleanClause query) { return query != null && query.getOccur() == BooleanClause.Occur.MUST; } private BooleanClause getTermQueryOrNull(Element element, String termStr) { if (termStr == null || termStr.isEmpty()) { return null; } final Query termQuery; final Term termQueryTerm = getTermQueryTerm(element, termStr); if (element.getNegation() || element.getMinOccurrence() == 0) { // we need to ignore this - negation, if any, must happen at the same position return null; } else if (element.isInflected() && element.isRegularExpression()) { Term lemmaQueryTerm = getQueryTerm(element, LEMMA_PREFIX + "(", simplifyRegex(termStr), ")"); final Query regexpQuery = getRegexQuery(lemmaQueryTerm, termStr, element); return new BooleanClause(regexpQuery, BooleanClause.Occur.MUST); } else if (element.isInflected() && !element.isRegularExpression()) { /* This is simpler, but leads to problem with e.g. German rules ZEITLICH_SYNCHRON and GEWISSEN_SUBST: Term lemmaQueryTerm = getQueryTerm(element, LEMMA_PREFIX, termStr, ""); Query query = new TermQuery(lemmaQueryTerm); return new BooleanClause(query, BooleanClause.Occur.MUST); */ final Synthesizer synthesizer = language.getSynthesizer(); if (synthesizer != null) { try { final String[] synthesized = synthesizer.synthesize(new AnalyzedToken(termStr, null, termStr), ".*", true); final Query query; if (synthesized.length == 0) { query = new TermQuery(termQueryTerm); } else { query = new RegexpQuery(getTermQueryTerm(element, StringUtils.join(synthesized, "|"))); } return new BooleanClause(query, BooleanClause.Occur.MUST); } catch (IOException e) { throw new RuntimeException( "Could not build Lucene query for '" + element + "' and '" + termStr + "'", e); } } return null; } else if (element.isRegularExpression()) { termQuery = getRegexQuery(termQueryTerm, termStr, element); } else { termQuery = new TermQuery(termQueryTerm); } return new BooleanClause(termQuery, BooleanClause.Occur.MUST); } // regex syntax not supported, but doesn't matter - remove or simplify it: private String simplifyRegex(String regex) { return regex.replace("(?:", "(").replace("\\d", "[0-9]").replace("\\w", "[a-zA-Z_0-9]"); } // the new and fast Regex query of Lucene doesn't support full Java regex syntax: private boolean needsSimplification(String regex) { return regex.contains("(?:") || regex.contains("\\d") || regex.contains("\\w"); } private BooleanClause getPosQueryOrNull(Element element, String pos) { if (pos == null || pos.isEmpty()) { return null; } final Query posQuery; final Term posQueryTerm; if (element.getPOSNegation() || element.getMinOccurrence() == 0) { // we need to ignore this - negation, if any, must happen at the same position return null; } else if (element.isPOStagRegularExpression()) { posQueryTerm = getQueryTerm(element, POS_PREFIX + "(", pos, ")"); posQuery = getRegexQuery(posQueryTerm, pos, element); } else { posQueryTerm = getQueryTerm(element, POS_PREFIX, pos, ""); posQuery = new TermQuery(posQueryTerm); } return new BooleanClause(posQuery, BooleanClause.Occur.MUST); } private Term getTermQueryTerm(Element element, String str) { if (element.isCaseSensitive()) { return new Term(FIELD_NAME, str); } else { return new Term(FIELD_NAME_LOWERCASE, str.toLowerCase()); } } private Term getQueryTerm(Element element, String prefix, String str, String suffix) { if (element.isCaseSensitive()) { return new Term(FIELD_NAME, prefix + str + suffix); } else { return new Term(FIELD_NAME_LOWERCASE, prefix.toLowerCase() + str.toLowerCase() + suffix.toLowerCase()); } } private Query getRegexQuery(Term term, String str, Element element) { try { if (needsSimplification(str)) { Term newTerm = new Term(term.field(), simplifyRegex(term.text())); return new RegexpQuery(newTerm); } if (str.contains("?iu") || str.contains("?-i")) { // Lucene's RegexpQuery doesn't seem to handle this correctly return getFallbackRegexQuery(str, element); } return new RegexpQuery(term); } catch (IllegalArgumentException e) { // fallback for special constructs like "\p{Punct}" not supported by Lucene RegExp: return getFallbackRegexQuery(str, element); } } /** Return a query with the old (and slow, but more complete) way Lucene implements regular expressions. */ private RegexQuery getFallbackRegexQuery(String str, Element element) { // No lowercase of str, so '\p{Punct}' doesn't become '\p{punct}': final RegexQuery query = new RegexQuery( new Term(element.isCaseSensitive() ? FIELD_NAME : FIELD_NAME_LOWERCASE, str)); query.setRegexImplementation( new JavaUtilRegexCapabilities(JavaUtilRegexCapabilities.FLAG_CASE_INSENSITIVE)); return query; } private void checkUnsupportedElement(Element patternElement) throws UnsupportedPatternRuleException { if (patternElement.hasOrGroup()) { // TODO: this is not enough: the first of the tokens in the <or> group will not get into this branch throw new UnsupportedPatternRuleException("<or> not yet supported."); } if (patternElement.isUnified()) { throw new UnsupportedPatternRuleException("Elements with unified tokens are not supported."); } if (patternElement.getString().matches("\\\\\\d+")) { // e.g. "\1" throw new UnsupportedPatternRuleException( "Elements with only match references (e.g. \\1) are not supported."); } } }