org.languagetool.dev.bigdata.GermanUppercasePhraseFinder.java Source code

Introduction

Here is the source code for org.languagetool.dev.bigdata.GermanUppercasePhraseFinder.java

Source

/* LanguageTool, a natural language style checker 
 * Copyright (C) 2016 Daniel Naber (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.bigdata;

import joptsimple.internal.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.*;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.languagetool.AnalyzedSentence;
import org.languagetool.AnalyzedTokenReadings;
import org.languagetool.JLanguageTool;
import org.languagetool.Languages;
import org.languagetool.tools.StringTools;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Prototype to find potential upper-only phrases like "Persischer Golf".
 * Uses Google 2-gram index.
 */
final class GermanUppercasePhraseFinder {

    private static final long MIN_TERM_LEN = 4;
    private static final long LIMIT = 500;

    private GermanUppercasePhraseFinder() {
    }

    public static void main(String[] args) throws IOException {
        if (args.length != 1) {
            System.out.println("Usage: " + GermanUppercasePhraseFinder.class.getSimpleName() + " <ngramIndexDir>");
            System.exit(1);
        }
        JLanguageTool lt = new JLanguageTool(Languages.getLanguageForShortCode("de"));
        FSDirectory fsDir = FSDirectory.open(new File(args[0]).toPath());
        IndexReader reader = DirectoryReader.open(fsDir);
        IndexSearcher searcher = new IndexSearcher(reader);
        Fields fields = MultiFields.getFields(reader);
        Terms terms = fields.terms("ngram");
        TermsEnum termsEnum = terms.iterator();
        int count = 0;
        BytesRef next;
        while ((next = termsEnum.next()) != null) {
            String term = next.utf8ToString();
            count++;
            //term = "persischer Golf";  // for testing
            String[] parts = term.split(" ");
            boolean useful = true;
            int lcCount = 0;
            List<String> ucParts = new ArrayList<>();
            for (String part : parts) {
                if (part.length() < MIN_TERM_LEN) {
                    useful = false;
                    break;
                }
                String uc = StringTools.uppercaseFirstChar(part);
                if (!part.equals(uc)) {
                    lcCount++;
                }
                ucParts.add(uc);
            }
            if (!useful || lcCount == 0 || lcCount == 2) {
                continue;
            }
            String uppercase = Strings.join(ucParts, " ");
            if (term.equals(uppercase)) {
                continue;
            }
            long thisCount = getOccurrenceCount(reader, searcher, term);
            long thisUpperCount = getOccurrenceCount(reader, searcher, uppercase);
            if (count % 10_000 == 0) {
                System.err.println(count + " @ " + term);
            }
            if (thisCount > LIMIT || thisUpperCount > LIMIT) {
                if (thisUpperCount > thisCount) {
                    if (isRelevant(lt, term)) {
                        float factor = (float) thisUpperCount / thisCount;
                        System.out.printf(
                                "%.2f " + thisUpperCount + " " + uppercase + " " + thisCount + " " + term + "\n",
                                factor);
                    }
                }
            }
        }
    }

    private static boolean isRelevant(JLanguageTool lt, String term) throws IOException {
        AnalyzedSentence analyzedSentence = lt.analyzeText(term).get(0);
        AnalyzedTokenReadings[] tokens = analyzedSentence.getTokensWithoutWhitespace();
        if (tokens.length == 1 + 2) { // 1 is for sentence start
            if (tokens[1].hasPartialPosTag("ADJ:") && tokens[2].hasPartialPosTag("SUB:")) {
                return true;
            }
        }
        return false;
    }

    private static long getOccurrenceCount(IndexReader reader, IndexSearcher searcher, String term)
            throws IOException {
        TopDocs topDocs = searcher.search(new TermQuery(new Term("ngram", term)), 5);
        if (topDocs.totalHits == 0) {
            return 0;
        }
        int docId = topDocs.scoreDocs[0].doc;
        Document document = reader.document(docId);
        return Long.parseLong(document.get("count"));
    }

}