Java tutorial
/******************************************************************************* * Copyright 2015-2016 - CNRS (Centre National de Recherche Scientifique) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * *******************************************************************************/ package eu.project.ttc.engines; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import org.apache.commons.lang.mutable.MutableInt; import org.apache.uima.UimaContext; import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.cas.FSIterator; import org.apache.uima.fit.component.JCasAnnotator_ImplBase; import org.apache.uima.fit.descriptor.ConfigurationParameter; import org.apache.uima.fit.descriptor.ExternalResource; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; import org.apache.uima.resource.ResourceInitializationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Functions; import com.google.common.base.Optional; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import eu.project.ttc.models.Term; import eu.project.ttc.models.TermIndex; import eu.project.ttc.models.TermOccurrence; import eu.project.ttc.models.VariationType; import eu.project.ttc.resources.TermIndexResource; import eu.project.ttc.types.SourceDocumentInformation; import eu.project.ttc.types.WordAnnotation; import eu.project.ttc.utils.JCasUtils; /** * Compiles and logs CAS stats. * * @author Damien Cram * */ public class CasStatCounter extends JCasAnnotator_ImplBase { private static final Logger LOGGER = LoggerFactory.getLogger(CasStatCounter.class); private Map<String, MutableInt> counters = Maps.newHashMap(); public static final String STAT_NAME = "StatName"; @ConfigurationParameter(name = STAT_NAME, mandatory = false) private String statName; public static final String DOCUMENT_PERIOD = "DocumentPeriod"; @ConfigurationParameter(name = DOCUMENT_PERIOD, mandatory = false, defaultValue = "-1") private int docPeriod; private boolean periodicStatEnabled = false; private int docIt; private long cumulatedFileSize; public static final String TO_TRACE_FILE = "ToTraceFile"; @ConfigurationParameter(name = TO_TRACE_FILE, mandatory = false) private String traceFileName; private Writer fileWriter; private static final String TSV_LINE_FORMAT = "%d\t%d\t%d\t%d\t%d\n"; @ExternalResource(key = TermIndexResource.TERM_INDEX, mandatory = true) private TermIndexResource termIndexResource; private Stopwatch sw; @Override public void initialize(UimaContext context) throws ResourceInitializationException { super.initialize(context); this.sw = Stopwatch.createStarted(); if (traceFileName != null) { File file = new File(traceFileName); try { this.fileWriter = new FileWriter(file); } catch (IOException e) { LOGGER.error("Could not create a writer to file {}", traceFileName); throw new ResourceInitializationException(e); } this.periodicStatEnabled = docPeriod > 0; LOGGER.info("Tracing time performance to file {}", file.getAbsolutePath()); } } @Override public void process(JCas aJCas) throws AnalysisEngineProcessException { this.docIt++; Optional<SourceDocumentInformation> sourceDocumentAnnotation = JCasUtils.getSourceDocumentAnnotation(aJCas); if (sourceDocumentAnnotation.isPresent()) this.cumulatedFileSize += sourceDocumentAnnotation.get().getDocumentSize(); FSIterator<Annotation> it = aJCas.getAnnotationIndex().iterator(); Annotation a; MutableInt i; while (it.hasNext()) { a = it.next(); i = counters.get(a.getType().getShortName()); if (i == null) counters.put(a.getType().getShortName(), new MutableInt(1)); else i.increment(); } if (periodicStatEnabled && this.docIt % this.docPeriod == 0) try { traceToFile(); } catch (IOException e) { throw new AnalysisEngineProcessException(e); } } private void traceToFile() throws IOException { String line = String.format(TSV_LINE_FORMAT, this.sw.elapsed(TimeUnit.MILLISECONDS), this.docIt, this.cumulatedFileSize, this.termIndexResource.getTermIndex().getTerms().size(), this.counters.get(WordAnnotation.class.getSimpleName()).intValue()); LOGGER.debug(line); this.fileWriter.write(line); this.fileWriter.flush(); } @Override protected void finalize() throws Throwable { this.fileWriter.close(); super.finalize(); } @Override public void collectionProcessComplete() throws AnalysisEngineProcessException { if (statName != null) logStats(); } private void logStats() { Ordering<String> a = Ordering.natural().reverse().onResultOf(Functions.forMap(counters)) .compound(Ordering.natural()); Map<String, MutableInt> map = ImmutableSortedMap.copyOf(counters, a); Iterator<Entry<String, MutableInt>> it = map.entrySet().iterator(); if (it.hasNext()) {// it will be empty if pipeline is run on empty collection Entry<String, MutableInt> mostFrequentAnno = it.next(); LOGGER.info("[{}] {}: {} ", statName, mostFrequentAnno.getKey(), mostFrequentAnno.getValue().intValue()); } int nbSyntacticVariants = 0; int nbMorphologicalVariants = 0; int nbGraphicalVariants = 0; int nbOccurrences = 0; int nbPrimaryOccOccurrences = 0; TermIndex tIndex = termIndexResource.getTermIndex(); for (Term t : tIndex.getTerms()) { nbMorphologicalVariants += Iterables.size(t.getVariations(VariationType.MORPHOLOGICAL)); nbSyntacticVariants += Iterables.size(t.getVariations(VariationType.SYNTACTICAL)); nbGraphicalVariants += Iterables.size(t.getVariations(VariationType.GRAPHICAL)); nbOccurrences += t.getOccurrences().size(); for (TermOccurrence o : t.getOccurrences()) { if (o.isPrimaryOccurrence()) nbPrimaryOccOccurrences++; } } // graphical variants are bidirectional nbGraphicalVariants /= 2; LOGGER.info("[{}] Nb terms: {} [sw: {}, mw: {}]", statName, tIndex.getTerms().size(), Iterators.size(tIndex.singleWordTermIterator()), Iterators.size(tIndex.multiWordTermIterator())); LOGGER.info("[{}] Nb words: {} [compounds: {}]", statName, tIndex.getWords().size(), Iterators.size(tIndex.compoundWordTermIterator())); LOGGER.info("[{}] Nb occurrences: {} [primary: {}]", statName, nbOccurrences, nbPrimaryOccOccurrences); LOGGER.info("[{}] Nb variants: {} [morph: {}, syn: {}, graph: {}]", statName, nbMorphologicalVariants + nbSyntacticVariants + nbGraphicalVariants, nbMorphologicalVariants, nbSyntacticVariants, nbGraphicalVariants); } }