Java tutorial
/** * Copyright 2007-2014 * Ubiquitous Knowledge Processing (UKP) Lab * Technische Universitt Darmstadt * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/. */ package de.tudarmstadt.ukp.dkpro.core.corenlp; import static org.apache.uima.util.Level.INFO; import static org.apache.uima.util.Level.WARNING; import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Properties; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.uima.UimaContext; import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.cas.CAS; import org.apache.uima.fit.component.JCasAnnotator_ImplBase; import org.apache.uima.fit.descriptor.ConfigurationParameter; import org.apache.uima.fit.descriptor.TypeCapability; import org.apache.uima.jcas.JCas; import org.apache.uima.resource.ResourceInitializationException; import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; import de.tudarmstadt.ukp.dkpro.core.api.metadata.SingletonTagset; import de.tudarmstadt.ukp.dkpro.core.api.parameter.ComponentParameters; import de.tudarmstadt.ukp.dkpro.core.api.resources.CasConfigurableProviderBase; import de.tudarmstadt.ukp.dkpro.core.api.resources.MappingProvider; import de.tudarmstadt.ukp.dkpro.core.api.resources.MappingProviderFactory; import de.tudarmstadt.ukp.dkpro.core.api.resources.ModelProviderBase; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.constituent.Constituent; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; import de.tudarmstadt.ukp.dkpro.core.corenlp.internal.DKPro2CoreNlp; import de.tudarmstadt.ukp.dkpro.core.corenlp.internal.CoreNlp2DKPro; import edu.stanford.nlp.parser.common.ParserGrammar; import edu.stanford.nlp.parser.lexparser.LexicalizedParser; import edu.stanford.nlp.parser.lexparser.Lexicon; import edu.stanford.nlp.parser.shiftreduce.BaseModel; import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser; import edu.stanford.nlp.pipeline.Annotation; import edu.stanford.nlp.pipeline.ParserAnnotator; import edu.stanford.nlp.process.PTBEscapingProcessor; import edu.stanford.nlp.trees.AbstractTreebankLanguagePack; import edu.stanford.nlp.trees.EnglishGrammaticalRelations; import edu.stanford.nlp.trees.EnglishGrammaticalStructureFactory; import edu.stanford.nlp.trees.GrammaticalRelation; import edu.stanford.nlp.trees.GrammaticalStructure; import edu.stanford.nlp.trees.GrammaticalStructureFactory; import edu.stanford.nlp.trees.TreebankLanguagePack; import edu.stanford.nlp.trees.UniversalEnglishGrammaticalRelations; import edu.stanford.nlp.trees.UniversalEnglishGrammaticalStructureFactory; import edu.stanford.nlp.trees.international.pennchinese.ChineseGrammaticalRelations; /** * Parser from CoreNLP. */ @TypeCapability(inputs = { "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence", "de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS" }, outputs = { "de.tudarmstadt.ukp.dkpro.core.api.syntax.type.constituent.Constituent", "de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency" }) public class CoreNlpParser extends JCasAnnotator_ImplBase { /** * Log the tag set(s) when a model is loaded. * * Default: {@code false} */ public static final String PARAM_PRINT_TAGSET = ComponentParameters.PARAM_PRINT_TAGSET; @ConfigurationParameter(name = PARAM_PRINT_TAGSET, mandatory = true, defaultValue = "false") private boolean printTagSet; /** * Use this language instead of the document language to resolve the model and tag set mapping. */ public static final String PARAM_LANGUAGE = ComponentParameters.PARAM_LANGUAGE; @ConfigurationParameter(name = PARAM_LANGUAGE, mandatory = false) private String language; /** * Variant of a model the model. Used to address a specific model if here are multiple models * for one language. */ public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) private String variant; /** * Location from which the model is read. */ public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = false) private String modelLocation; /** * The character encoding used by the model. */ public static final String PARAM_MODEL_ENCODING = ComponentParameters.PARAM_MODEL_ENCODING; @ConfigurationParameter(name = PARAM_MODEL_ENCODING, mandatory = false) private String modelEncoding; /** * Location of the mapping file for dependency tags to UIMA types. */ public static final String PARAM_DEPENDENCY_MAPPING_LOCATION = ComponentParameters.PARAM_DEPENDENCY_MAPPING_LOCATION; @ConfigurationParameter(name = PARAM_DEPENDENCY_MAPPING_LOCATION, mandatory = false) private String dependencyMappingLocation; /** * Location of the mapping file for dependency tags to UIMA types. */ public static final String PARAM_CONSTITUENT_MAPPING_LOCATION = ComponentParameters.PARAM_CONSTITUENT_MAPPING_LOCATION; @ConfigurationParameter(name = PARAM_CONSTITUENT_MAPPING_LOCATION, mandatory = false) private String constituentMappingLocation; /** * Location of the mapping file for part-of-speech tags to UIMA types. */ public static final String PARAM_POS_MAPPING_LOCATION = ComponentParameters.PARAM_POS_MAPPING_LOCATION; @ConfigurationParameter(name = PARAM_POS_MAPPING_LOCATION, mandatory = false) private String posMappingLocation; /** * Use the {@link String#intern()} method on tags. This is usually a good idea to avoid * spaming the heap with thousands of strings representing only a few different tags. * * Default: {@code false} */ public static final String PARAM_INTERN_TAGS = ComponentParameters.PARAM_INTERN_TAGS; @ConfigurationParameter(name = PARAM_INTERN_TAGS, mandatory = false, defaultValue = "true") private boolean internStrings; public static final String PARAM_MAX_SENTENCE_LENGTH = ComponentParameters.PARAM_MAX_SENTENCE_LENGTH; @ConfigurationParameter(name = PARAM_MAX_SENTENCE_LENGTH, mandatory = true, defaultValue = "2147483647") private int maxSentenceLength; public static final String PARAM_NUM_THREADS = ComponentParameters.PARAM_NUM_THREADS; @ConfigurationParameter(name = PARAM_NUM_THREADS, mandatory = true, defaultValue = ComponentParameters.AUTO_NUM_THREADS) private int numThreads; public static final String PARAM_MAX_TIME = "maxTime"; @ConfigurationParameter(name = PARAM_MAX_TIME, mandatory = true, defaultValue = "-1") private int maxTime; /** * Enable all traditional PTB3 token transforms (like -LRB-, -RRB-). * * @see PTBEscapingProcessor */ public static final String PARAM_PTB3_ESCAPING = "ptb3Escaping"; @ConfigurationParameter(name = PARAM_PTB3_ESCAPING, mandatory = true, defaultValue = "true") private boolean ptb3Escaping; /** * List of extra token texts (usually single character strings) that should be treated like * opening quotes and escaped accordingly before being sent to the parser. */ public static final String PARAM_QUOTE_BEGIN = "quoteBegin"; @ConfigurationParameter(name = PARAM_QUOTE_BEGIN, mandatory = false) private List<String> quoteBegin; /** * List of extra token texts (usually single character strings) that should be treated like * closing quotes and escaped accordingly before being sent to the parser. */ public static final String PARAM_QUOTE_END = "quoteEnd"; @ConfigurationParameter(name = PARAM_QUOTE_END, mandatory = false) private List<String> quoteEnd; public static final String PARAM_EXTRA_DEPENDENCIES = "extraDependencies"; @ConfigurationParameter(name = PARAM_EXTRA_DEPENDENCIES, mandatory = true, defaultValue = "NONE") GrammaticalStructure.Extras extraDependencies; /** * Sets whether to create or not to create constituent tags. This is required for POS-tagging * and lemmatization. * <p> * Default: {@code true} */ public static final String PARAM_WRITE_CONSTITUENT = ComponentParameters.PARAM_WRITE_CONSTITUENT; @ConfigurationParameter(name = PARAM_WRITE_CONSTITUENT, mandatory = true, defaultValue = "true") private boolean writeConstituent; /** * If this parameter is set to true, each sentence is annotated with a PennTree-Annotation, * containing the whole parse tree in Penn Treebank style format. * <p> * Default: {@code false} */ public static final String PARAM_WRITE_PENN_TREE = ComponentParameters.PARAM_WRITE_PENN_TREE; @ConfigurationParameter(name = PARAM_WRITE_PENN_TREE, mandatory = true, defaultValue = "false") private boolean writePennTree; /** * Sets whether to use or not to use existing POS tags. * <p> * Default: {@code true} */ public static final String PARAM_READ_POS = ComponentParameters.PARAM_READ_POS; @ConfigurationParameter(name = PARAM_READ_POS, mandatory = true, defaultValue = "true") private boolean readPos; /** * Sets whether to create or not to create POS tags. The creation of constituent tags must be * turned on for this to work. * <p> * Default: {@code false} */ public static final String PARAM_WRITE_POS = ComponentParameters.PARAM_WRITE_POS; @ConfigurationParameter(name = PARAM_WRITE_POS, mandatory = true, defaultValue = "false") private boolean writePos; /** * Sets whether to create or not to create dependency annotations. * * <p>Default: {@code true} */ public static final String PARAM_WRITE_DEPENDENCY = ComponentParameters.PARAM_WRITE_DEPENDENCY; @ConfigurationParameter(name = PARAM_WRITE_DEPENDENCY, mandatory = true, defaultValue = "true") private boolean writeDependency; public static final String PARAM_ORIGINAL_DEPENDENCIES = "originalDependencies"; @ConfigurationParameter(name = PARAM_ORIGINAL_DEPENDENCIES, mandatory = true, defaultValue = "true") private boolean originalDependencies; // CoreNlpParser PARAM_KEEP_PUNCTUATION has no effect #965 public static final String PARAM_KEEP_PUNCTUATION = "keepPunctuation"; @ConfigurationParameter(name = PARAM_KEEP_PUNCTUATION, mandatory = true, defaultValue = "false") private boolean keepPunctuation; private CasConfigurableProviderBase<ParserAnnotator> annotatorProvider; private MappingProvider dependencyMappingProvider; private MappingProvider constituentMappingProvider; private MappingProvider posMappingProvider; @Override public void initialize(UimaContext aContext) throws ResourceInitializationException { super.initialize(aContext); annotatorProvider = new CoreNlpParserModelProvider(this); constituentMappingProvider = MappingProviderFactory .createConstituentMappingProvider(constituentMappingLocation, language, annotatorProvider); dependencyMappingProvider = MappingProviderFactory .createDependencyMappingProvider(dependencyMappingLocation, language, annotatorProvider); posMappingProvider = MappingProviderFactory.createPosMappingProvider(posMappingLocation, language, annotatorProvider); numThreads = ComponentParameters.computeNumThreads(numThreads); } @Override public void process(JCas aJCas) throws AnalysisEngineProcessException { CAS cas = aJCas.getCas(); annotatorProvider.configure(cas); // Transfer from CAS to CoreNLP DKPro2CoreNlp converter = new DKPro2CoreNlp(); converter.setPtb3Escaping(ptb3Escaping); converter.setQuoteBegin(quoteBegin); converter.setQuoteEnd(quoteEnd); converter.setEncoding(modelEncoding); converter.setReadPos(readPos); Annotation document = new Annotation((String) null); converter.convert(aJCas, document); // Actual processing ParserAnnotator annotator = annotatorProvider.getResource(); annotator.annotate(document); // Get TreebankLanguagePack ParserGrammar parser; try { parser = (ParserGrammar) FieldUtils.readField(annotator, "parser", true); } catch (IllegalAccessException e) { throw new AnalysisEngineProcessException(e); } TreebankLanguagePack tlp = parser.getTLPParams().treebankLanguagePack(); // Transfer back into the CAS if (writePos) { posMappingProvider.configure(cas); CoreNlp2DKPro.convertPOSs(aJCas, document, posMappingProvider, internStrings); } if (writeConstituent) { constituentMappingProvider.configure(cas); CoreNlp2DKPro.convertConstituents(aJCas, document, constituentMappingProvider, internStrings, tlp); } if (writePennTree) { CoreNlp2DKPro.convertPennTree(aJCas, document); } if (writeDependency) { dependencyMappingProvider.configure(cas); CoreNlp2DKPro.convertDependencies(aJCas, document, dependencyMappingProvider, internStrings); } } private class CoreNlpParserModelProvider extends ModelProviderBase<ParserAnnotator> { public CoreNlpParserModelProvider(Object aObject) { super(aObject, "stanfordnlp", "parser"); // setDefault(PACKAGE, "de/tudarmstadt/ukp/dkpro/core/stanfordnlp"); setDefault(LOCATION, "classpath:/de/tudarmstadt/ukp/dkpro/core/stanfordnlp/lib/parser-${language}-${variant}.properties"); } @SuppressWarnings("unchecked") @Override protected ParserAnnotator produceResource(URL aUrl) throws IOException { String modelFile = aUrl.toString(); // Loading gzipped files from URL is broken in CoreNLP // https://github.com/stanfordnlp/CoreNLP/issues/94 if (modelFile.startsWith("jar:") && modelFile.endsWith(".gz")) { modelFile = org.apache.commons.lang.StringUtils.substringAfter(modelFile, "!/"); } Properties coreNlpProps = new Properties(); // Need to set annotators because CoreNLP checks for the presence of the sentiment // annotator to decide the default value for "parse.binaryTrees" coreNlpProps.setProperty("annotators", ""); coreNlpProps.setProperty("parse.model", modelFile); // coreNlpProps.setProperty("parse.flags", ...); coreNlpProps.setProperty("parse.maxlen", Integer.toString(maxSentenceLength)); coreNlpProps.setProperty("parse.kbest", Integer.toString(3)); // CoreNlpParser PARAM_KEEP_PUNCTUATION has no effect #965 coreNlpProps.setProperty("parse.keepPunct", Boolean.toString(keepPunctuation)); // coreNlpProps.setProperty("parse.treemap", ...); coreNlpProps.setProperty("parse.maxtime", Integer.toString(maxTime)); coreNlpProps.setProperty("parse.buildgraphs", Boolean.toString(writeDependency)); coreNlpProps.setProperty("parse.originalDependencies", Boolean.toString(originalDependencies)); coreNlpProps.setProperty("parse.nthreads", Integer.toString(numThreads)); // coreNlpProps.setProperty("parse.binaryTrees", ...); // coreNlpProps.setProperty("parse.nosquash", ...); coreNlpProps.setProperty("parse.extradependencies", extraDependencies.toString()); ParserAnnotator annotator = new ParserAnnotator("parse", coreNlpProps); getLogger().info(ParserAnnotator.signature("parse", coreNlpProps)); ParserGrammar parser; try { parser = (ParserGrammar) FieldUtils.readField(annotator, "parser", true); } catch (IllegalAccessException e) { throw new IOException(e); } Properties metadata = getResourceMetaData(); AbstractTreebankLanguagePack lp = (AbstractTreebankLanguagePack) parser.getTLPParams() .treebankLanguagePack(); // https://mailman.stanford.edu/pipermail/parser-user/2012-November/002117.html // The tagIndex does give all and only the set of POS tags used in the // current grammar. However, these are the split tags actually used by the // grammar. If you really want the user-visible non-split tags of the // original treebank, then you'd need to map them all through the // op.treebankLanguagePack().basicCategory(). -- C. Manning SingletonTagset posTags = new SingletonTagset(POS.class, metadata.getProperty("pos.tagset")); if (parser instanceof LexicalizedParser) { LexicalizedParser lexParser = (LexicalizedParser) parser; for (String tag : lexParser.tagIndex) { String t = lp.basicCategory(tag); // Strip grammatical function from tag int gfIdx = t.indexOf(lp.getGfCharacter()); if (gfIdx > 0) { // TODO should collect syntactic functions in separate tagset // syntacticFunction = nodeLabelValue.substring(gfIdx + 1); t = t.substring(0, gfIdx); } posTags.add(lp.basicCategory(t)); } posTags.remove(Lexicon.BOUNDARY_TAG); addTagset(posTags, writePos); } // https://mailman.stanford.edu/pipermail/parser-user/2012-November/002117.html // For constituent categories, there isn't an index of just them. The // stateIndex has both constituent categories and POS tags in it, so you'd // need to set difference out the tags from the tagIndex, and then it's as // above. -- C. Manning SingletonTagset constTags = new SingletonTagset(Constituent.class, metadata.getProperty("constituent.tagset")); Iterable<String> states; if (parser instanceof LexicalizedParser) { states = ((LexicalizedParser) parser).stateIndex; } else if (parser instanceof ShiftReduceParser) { try { BaseModel model = (BaseModel) FieldUtils.readField(parser, "model", true); states = (Iterable<String>) FieldUtils.readField(model, "knownStates", true); // states = ((ShiftReduceParser) pd).tagSet(); } catch (IllegalAccessException e) { throw new IOException(e); } } else { throw new IllegalStateException("Unknown parser type [" + parser.getClass().getName() + "]"); } for (String tag : states) { String t = lp.basicCategory(tag); // https://mailman.stanford.edu/pipermail/parser-user/2012-December/002156.html // The parser algorithm used is a binary parser, so what we do is // binarize trees by turning A -> B, C, D into A -> B, @A, @A -> C, D. // (That's roughly how it goes, although the exact details are somewhat // different.) When parsing, we parse to a binarized tree and then // unbinarize it before returning. That's the origin of the @ classes. // -- J. Bauer if (!t.startsWith("@")) { // Strip grammatical function from tag int gfIdx = t.indexOf(lp.getGfCharacter()); if (gfIdx > 0) { // TODO should collect syntactic functions in separate tagset // syntacticFunction = nodeLabelValue.substring(gfIdx + 1); t = t.substring(0, gfIdx); } if (t.length() > 0) { constTags.add(t); } } } constTags.remove(Lexicon.BOUNDARY_TAG); constTags.removeAll(posTags); if (writeConstituent) { addTagset(constTags); } // There is no way to determine the relations via the GrammaticalStructureFactory // API, so we do it manually here for the languages known to support this. GrammaticalStructureFactory gsf = null; try { gsf = lp.grammaticalStructureFactory(lp.punctuationWordRejectFilter(), lp.typedDependencyHeadFinder()); } catch (UnsupportedOperationException e) { getContext().getLogger().log(WARNING, "Current model does not seem to support " + "dependencies."); } // TODO: Consider whether r.getShortName() or r.toString() is the right one to use // here. Cf. // https://mailman.stanford.edu/pipermail/java-nlp-user/2016-January/007417.html // https://mailman.stanford.edu/pipermail/java-nlp-user/2013-December/004429.html if (gsf != null && EnglishGrammaticalStructureFactory.class.equals(gsf.getClass())) { SingletonTagset depTags = new SingletonTagset(Dependency.class, "stanford341"); for (GrammaticalRelation r : EnglishGrammaticalRelations.values()) { depTags.add(r.getShortName()); } if (writeDependency) { addTagset(depTags, writeDependency); } } else if (gsf != null && UniversalEnglishGrammaticalStructureFactory.class.equals(gsf.getClass())) { SingletonTagset depTags = new SingletonTagset(Dependency.class, "universal"); for (GrammaticalRelation r : UniversalEnglishGrammaticalRelations.values()) { depTags.add(r.getShortName()); } if (writeDependency) { addTagset(depTags, writeDependency); } } else if (gsf != null && ChineseGrammaticalRelations.class.equals(gsf.getClass())) { SingletonTagset depTags = new SingletonTagset(Dependency.class, "stanford"); for (GrammaticalRelation r : ChineseGrammaticalRelations.values()) { depTags.add(r.getShortName()); } if (writeDependency) { addTagset(depTags, writeDependency); } } if (printTagSet) { getContext().getLogger().log(INFO, getTagset().toString()); } return annotator; } } }