Java tutorial
/******************************************************************************* * Copyright 2015-2016 - CNRS (Centre National de Recherche Scientifique) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * *******************************************************************************/ package eu.project.ttc.engines; import java.util.Collection; import java.util.List; import java.util.Timer; import java.util.TimerTask; import org.apache.uima.UimaContext; import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.fit.component.JCasAnnotator_ImplBase; import org.apache.uima.fit.descriptor.ConfigurationParameter; import org.apache.uima.fit.descriptor.ExternalResource; import org.apache.uima.jcas.JCas; import org.apache.uima.resource.ResourceInitializationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.math.IntMath; import eu.project.ttc.engines.desc.Lang; import eu.project.ttc.history.TermHistoryResource; import eu.project.ttc.metrics.DiacriticInsensitiveLevenshtein; import eu.project.ttc.metrics.EditDistance; import eu.project.ttc.models.Term; import eu.project.ttc.models.TermIndex; import eu.project.ttc.models.TermWord; import eu.project.ttc.models.VariationType; import eu.project.ttc.models.Word; import eu.project.ttc.models.index.AbstractTermValueProvider; import eu.project.ttc.models.index.CustomTermIndex; import eu.project.ttc.models.index.TermValueProvider; import eu.project.ttc.resources.ObserverResource; import eu.project.ttc.resources.ObserverResource.SubTaskObserver; import eu.project.ttc.resources.TermIndexResource; import eu.project.ttc.utils.StringUtils; /** * * Gather terms whe their edit distances (ignoring diacritics) are * under certain threshold (not normalized). * * @author Damien Cram * */ public class GraphicalVariantGatherer extends JCasAnnotator_ImplBase { private static final Logger logger = LoggerFactory.getLogger(GraphicalVariantGatherer.class); public static final String TASK_NAME = "Computing graphical variants"; private static final int OBSERVER_STEP = 10000; private static final char JOIN_CHAR = ':'; @ExternalResource(key = ObserverResource.OBSERVER, mandatory = false) protected ObserverResource observerResource; @ExternalResource(key = TermIndexResource.TERM_INDEX, mandatory = true) private TermIndexResource termIndexResource; @ExternalResource(key = TermHistoryResource.TERM_HISTORY, mandatory = true) private TermHistoryResource historyResource; public static final String LANG = "lang"; @ConfigurationParameter(name = LANG, mandatory = true) private String lang; public static final String SIMILARITY_THRESHOLD = "SimilarityThreshold"; @ConfigurationParameter(name = SIMILARITY_THRESHOLD, mandatory = true) private float threshold; private EditDistance distance; private Lang language; /* * the n first letter that serves as index */ private int n = 2; private int totalComparisons = 0; private int nbComparisons = 0; private TermValueProvider nFirstLettersProvider = new AbstractTermValueProvider("") { @Override public Collection<String> getClasses(TermIndex termIndex, Term term) { if (term.getWords().size() == 1) { Word word = term.getWords().get(0).getWord(); if (word.getLemma().length() < 5) return ImmutableList.of(); else { String substring = StringUtils .replaceAccents(word.getLemma().toLowerCase(language.getLocale()).substring(0, 4)); return ImmutableList.of(substring); } } StringBuilder builder = new StringBuilder(); String normalizedStem; int i = 0; for (TermWord tw : term.getWords()) { if (i > 0) { builder.append(JOIN_CHAR); } normalizedStem = tw.getWord().getNormalizedStem(); if (normalizedStem.length() > n) builder.append(normalizedStem.substring(0, n).toLowerCase(language.getLocale())); else builder.append(normalizedStem.toLowerCase(language.getLocale())); i++; } if (builder.length() >= n) return ImmutableList.of(builder.toString()); else return ImmutableList.of(); } }; @Override public void initialize(UimaContext context) throws ResourceInitializationException { super.initialize(context); this.language = Lang.forName(lang); distance = new DiacriticInsensitiveLevenshtein(this.language.getLocale()); if (observerResource != null) taskObserver = Optional.of(observerResource.getTaskObserver(TASK_NAME)); } @Override public void process(JCas aJCas) throws AnalysisEngineProcessException { } private Optional<SubTaskObserver> taskObserver = Optional.absent(); @Override public void collectionProcessComplete() throws AnalysisEngineProcessException { logger.info("Starting graphical term gathering for TermIndex {}", this.termIndexResource.getTermIndex().getName()); if (termIndexResource.getTermIndex().getTerms().isEmpty()) return; // create the index String indexName = String.format("_%d_first_letters_", n); final TermIndex termIndex = this.termIndexResource.getTermIndex(); CustomTermIndex customIndex = termIndex.createCustomIndex(indexName, nFirstLettersProvider); // clean singleton classes logger.debug("Cleaning singleton keys"); customIndex.cleanSingletonKeys(); logger.debug("Graphical gathering over {} classes", customIndex.size()); // get the total number of comparisons for (String key : customIndex.keySet()) totalComparisons += IntMath.binomial(customIndex.getTerms(key).size(), 2); if (taskObserver.isPresent()) taskObserver.get().setTotalTaskWork(totalComparisons); logger.debug("Number of distance edition pairs to compute: {}", totalComparisons); // Log the progress every 5 seconds Timer progressLoggerTimer = new Timer("Syn. Variant Gathering Timer"); progressLoggerTimer.schedule(new TimerTask() { @Override public void run() { GraphicalVariantGatherer.logger.info("progress for graphical variant gathering: {}%", String.format("%.2f", ((float) nbComparisons * 100) / totalComparisons)); } }, 5000l, 5000l); // do the distance computation on each class List<Term> terms; Term t1, t2; int i, j; int gatheredCnt = 0; double dist = 0d; for (String key : customIndex.keySet()) { terms = customIndex.getTerms(key); logger.trace("Graphical gathering over term class {} of size: {}", key, terms.size()); for (i = 0; i < terms.size(); i++) { t1 = terms.get(i); for (j = i + 1; j < terms.size(); j++) { nbComparisons++; if (nbComparisons % OBSERVER_STEP == 0 && taskObserver.isPresent()) taskObserver.get().work(OBSERVER_STEP); t2 = terms.get(j); dist = distance.computeNormalized(t1.getLemma(), t2.getLemma()); if (dist >= this.threshold) { gatheredCnt++; if (t1.getLemma().compareTo(t2.getLemma()) <= 0) t1.addTermVariation(t2, VariationType.GRAPHICAL, dist); else t2.addTermVariation(t1, VariationType.GRAPHICAL, dist); watch(t1, t2, dist); watch(t2, t1, dist); } } } } // log some stats logger.debug("Graphical gathering {} terms gathered / {} pairs compared", gatheredCnt, nbComparisons); // free memory taken by the index termIndex.dropCustomIndex(indexName); progressLoggerTimer.cancel(); } private void watch(Term t1, Term t2, double dist) { if (historyResource.getHistory().isWatched(t1.getGroupingKey())) historyResource.getHistory().saveEvent(t1.getGroupingKey(), this.getClass(), "Term has a new graphical variant " + t2 + " (dist=" + dist + ")"); } }