Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.analysis.hunspell; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.IntsRefBuilder; import org.apache.lucene.util.OfflineSorter.ByteSequencesReader; import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter; import org.apache.lucene.util.OfflineSorter; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.CharSequenceOutputs; import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.IntSequenceOutputs; import org.apache.lucene.util.fst.Outputs; import org.apache.lucene.util.fst.Util; /** * In-memory structure for the dictionary (.dic) and affix (.aff) * data of a hunspell dictionary. */ public class Dictionary { static final char[] NOFLAGS = new char[0]; private static final String ALIAS_KEY = "AF"; private static final String MORPH_ALIAS_KEY = "AM"; private static final String PREFIX_KEY = "PFX"; private static final String SUFFIX_KEY = "SFX"; private static final String FLAG_KEY = "FLAG"; private static final String COMPLEXPREFIXES_KEY = "COMPLEXPREFIXES"; private static final String CIRCUMFIX_KEY = "CIRCUMFIX"; private static final String IGNORE_KEY = "IGNORE"; private static final String ICONV_KEY = "ICONV"; private static final String OCONV_KEY = "OCONV"; private static final String FULLSTRIP_KEY = "FULLSTRIP"; private static final String LANG_KEY = "LANG"; private static final String KEEPCASE_KEY = "KEEPCASE"; private static final String NEEDAFFIX_KEY = "NEEDAFFIX"; private static final String PSEUDOROOT_KEY = "PSEUDOROOT"; private static final String ONLYINCOMPOUND_KEY = "ONLYINCOMPOUND"; private static final String NUM_FLAG_TYPE = "num"; private static final String UTF8_FLAG_TYPE = "UTF-8"; private static final String LONG_FLAG_TYPE = "long"; // TODO: really for suffixes we should reverse the automaton and run them backwards private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*"; private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s"; FST<IntsRef> prefixes; FST<IntsRef> suffixes; // all condition checks used by prefixes and suffixes. these are typically re-used across // many affix stripping rules. so these are deduplicated, to save RAM. ArrayList<CharacterRunAutomaton> patterns = new ArrayList<>(); // the entries in the .dic file, mapping to their set of flags. // the fst output is the ordinal list for flagLookup FST<IntsRef> words; // the list of unique flagsets (wordforms). theoretically huge, but practically // small (e.g. for polish this is 756), otherwise humans wouldn't be able to deal with it either. BytesRefHash flagLookup = new BytesRefHash(); // the list of unique strip affixes. char[] stripData; int[] stripOffsets; // 8 bytes per affix byte[] affixData = new byte[64]; private int currentAffix = 0; private FlagParsingStrategy flagParsingStrategy = new SimpleFlagParsingStrategy(); // Default flag parsing strategy // AF entries private String[] aliases; private int aliasCount = 0; // AM entries private String[] morphAliases; private int morphAliasCount = 0; // st: morphological entries (either directly, or aliased from AM) private String[] stemExceptions = new String[8]; private int stemExceptionCount = 0; // we set this during sorting, so we know to add an extra FST output. // when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions boolean hasStemExceptions; private final Path tempPath = getDefaultTempDir(); // TODO: make this configurable? boolean ignoreCase; boolean complexPrefixes; boolean twoStageAffix; // if no affixes have continuation classes, no need to do 2-level affix stripping int circumfix = -1; // circumfix flag, or -1 if one is not defined int keepcase = -1; // keepcase flag, or -1 if one is not defined int needaffix = -1; // needaffix flag, or -1 if one is not defined int onlyincompound = -1; // onlyincompound flag, or -1 if one is not defined // ignored characters (dictionary, affix, inputs) private char[] ignore; // FSTs used for ICONV/OCONV, output ord pointing to replacement text FST<CharsRef> iconv; FST<CharsRef> oconv; boolean needsInputCleaning; boolean needsOutputCleaning; // true if we can strip suffixes "down to nothing" boolean fullStrip; // language declaration of the dictionary String language; // true if case algorithms should use alternate (Turkish/Azeri) mapping boolean alternateCasing; /** * Creates a new Dictionary containing the information read from the provided InputStreams to hunspell affix * and dictionary files. * You have to close the provided InputStreams yourself. * * @param tempDir Directory to use for offline sorting * @param tempFileNamePrefix prefix to use to generate temp file names * @param affix InputStream for reading the hunspell affix file (won't be closed). * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed). * @throws IOException Can be thrown while reading from the InputStreams * @throws ParseException Can be thrown if the content of the files does not meet expected formats */ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, InputStream dictionary) throws IOException, ParseException { this(tempDir, tempFileNamePrefix, affix, Collections.singletonList(dictionary), false); } /** * Creates a new Dictionary containing the information read from the provided InputStreams to hunspell affix * and dictionary files. * You have to close the provided InputStreams yourself. * * @param tempDir Directory to use for offline sorting * @param tempFileNamePrefix prefix to use to generate temp file names * @param affix InputStream for reading the hunspell affix file (won't be closed). * @param dictionaries InputStream for reading the hunspell dictionary files (won't be closed). * @throws IOException Can be thrown while reading from the InputStreams * @throws ParseException Can be thrown if the content of the files does not meet expected formats */ public Dictionary(Directory tempDir, String tempFileNamePrefix, InputStream affix, List<InputStream> dictionaries, boolean ignoreCase) throws IOException, ParseException { this.ignoreCase = ignoreCase; this.needsInputCleaning = ignoreCase; this.needsOutputCleaning = false; // set if we have an OCONV flagLookup.add(new BytesRef()); // no flags -> ord 0 Path aff = Files.createTempFile(tempPath, "affix", "aff"); OutputStream out = new BufferedOutputStream(Files.newOutputStream(aff)); InputStream aff1 = null; InputStream aff2 = null; boolean success = false; try { // copy contents of affix stream to temp file final byte[] buffer = new byte[1024 * 8]; int len; while ((len = affix.read(buffer)) > 0) { out.write(buffer, 0, len); } out.close(); // pass 1: get encoding aff1 = new BufferedInputStream(Files.newInputStream(aff)); String encoding = getDictionaryEncoding(aff1); // pass 2: parse affixes CharsetDecoder decoder = getJavaEncoding(encoding); aff2 = new BufferedInputStream(Files.newInputStream(aff)); readAffixFile(aff2, decoder); // read dictionary entries IntSequenceOutputs o = IntSequenceOutputs.getSingleton(); Builder<IntsRef> b = new Builder<>(FST.INPUT_TYPE.BYTE4, o); readDictionaryFiles(tempDir, tempFileNamePrefix, dictionaries, decoder, b); words = b.finish(); aliases = null; // no longer needed morphAliases = null; // no longer needed success = true; } finally { IOUtils.closeWhileHandlingException(out, aff1, aff2); if (success) { Files.delete(aff); } else { IOUtils.deleteFilesIgnoringExceptions(aff); } } } /** * Looks up Hunspell word forms from the dictionary */ IntsRef lookupWord(char word[], int offset, int length) { return lookup(words, word, offset, length); } // only for testing IntsRef lookupPrefix(char word[], int offset, int length) { return lookup(prefixes, word, offset, length); } // only for testing IntsRef lookupSuffix(char word[], int offset, int length) { return lookup(suffixes, word, offset, length); } IntsRef lookup(FST<IntsRef> fst, char word[], int offset, int length) { if (fst == null) { return null; } final FST.BytesReader bytesReader = fst.getBytesReader(); final FST.Arc<IntsRef> arc = fst.getFirstArc(new FST.Arc<IntsRef>()); // Accumulate output as we go final IntsRef NO_OUTPUT = fst.outputs.getNoOutput(); IntsRef output = NO_OUTPUT; int l = offset + length; try { for (int i = offset, cp = 0; i < l; i += Character.charCount(cp)) { cp = Character.codePointAt(word, i, l); if (fst.findTargetArc(cp, arc, arc, bytesReader) == null) { return null; } else if (arc.output != NO_OUTPUT) { output = fst.outputs.add(output, arc.output); } } if (fst.findTargetArc(FST.END_LABEL, arc, arc, bytesReader) == null) { return null; } else if (arc.output != NO_OUTPUT) { return fst.outputs.add(output, arc.output); } else { return output; } } catch (IOException bogus) { throw new RuntimeException(bogus); } } /** * Reads the affix file through the provided InputStream, building up the prefix and suffix maps * * @param affixStream InputStream to read the content of the affix file from * @param decoder CharsetDecoder to decode the content of the file * @throws IOException Can be thrown while reading from the InputStream */ private void readAffixFile(InputStream affixStream, CharsetDecoder decoder) throws IOException, ParseException { TreeMap<String, List<Integer>> prefixes = new TreeMap<>(); TreeMap<String, List<Integer>> suffixes = new TreeMap<>(); Map<String, Integer> seenPatterns = new HashMap<>(); // zero condition -> 0 ord seenPatterns.put(".*", 0); patterns.add(null); // zero strip -> 0 ord Map<String, Integer> seenStrips = new LinkedHashMap<>(); seenStrips.put("", 0); LineNumberReader reader = new LineNumberReader(new InputStreamReader(affixStream, decoder)); String line = null; while ((line = reader.readLine()) != null) { // ignore any BOM marker on first line if (reader.getLineNumber() == 1 && line.startsWith("\uFEFF")) { line = line.substring(1); } if (line.startsWith(ALIAS_KEY)) { parseAlias(line); } else if (line.startsWith(MORPH_ALIAS_KEY)) { parseMorphAlias(line); } else if (line.startsWith(PREFIX_KEY)) { parseAffix(prefixes, line, reader, PREFIX_CONDITION_REGEX_PATTERN, seenPatterns, seenStrips); } else if (line.startsWith(SUFFIX_KEY)) { parseAffix(suffixes, line, reader, SUFFIX_CONDITION_REGEX_PATTERN, seenPatterns, seenStrips); } else if (line.startsWith(FLAG_KEY)) { // Assume that the FLAG line comes before any prefix or suffixes // Store the strategy so it can be used when parsing the dic file flagParsingStrategy = getFlagParsingStrategy(line); } else if (line.equals(COMPLEXPREFIXES_KEY)) { complexPrefixes = true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix } else if (line.startsWith(CIRCUMFIX_KEY)) { String parts[] = line.split("\\s+"); if (parts.length != 2) { throw new ParseException("Illegal CIRCUMFIX declaration", reader.getLineNumber()); } circumfix = flagParsingStrategy.parseFlag(parts[1]); } else if (line.startsWith(KEEPCASE_KEY)) { String parts[] = line.split("\\s+"); if (parts.length != 2) { throw new ParseException("Illegal KEEPCASE declaration", reader.getLineNumber()); } keepcase = flagParsingStrategy.parseFlag(parts[1]); } else if (line.startsWith(NEEDAFFIX_KEY) || line.startsWith(PSEUDOROOT_KEY)) { String parts[] = line.split("\\s+"); if (parts.length != 2) { throw new ParseException("Illegal NEEDAFFIX declaration", reader.getLineNumber()); } needaffix = flagParsingStrategy.parseFlag(parts[1]); } else if (line.startsWith(ONLYINCOMPOUND_KEY)) { String parts[] = line.split("\\s+"); if (parts.length != 2) { throw new ParseException("Illegal ONLYINCOMPOUND declaration", reader.getLineNumber()); } onlyincompound = flagParsingStrategy.parseFlag(parts[1]); } else if (line.startsWith(IGNORE_KEY)) { String parts[] = line.split("\\s+"); if (parts.length != 2) { throw new ParseException("Illegal IGNORE declaration", reader.getLineNumber()); } ignore = parts[1].toCharArray(); Arrays.sort(ignore); needsInputCleaning = true; } else if (line.startsWith(ICONV_KEY) || line.startsWith(OCONV_KEY)) { String parts[] = line.split("\\s+"); String type = parts[0]; if (parts.length != 2) { throw new ParseException("Illegal " + type + " declaration", reader.getLineNumber()); } int num = Integer.parseInt(parts[1]); FST<CharsRef> res = parseConversions(reader, num); if (type.equals("ICONV")) { iconv = res; needsInputCleaning |= iconv != null; } else { oconv = res; needsOutputCleaning |= oconv != null; } } else if (line.startsWith(FULLSTRIP_KEY)) { fullStrip = true; } else if (line.startsWith(LANG_KEY)) { language = line.substring(LANG_KEY.length()).trim(); alternateCasing = "tr_TR".equals(language) || "az_AZ".equals(language); } } this.prefixes = affixFST(prefixes); this.suffixes = affixFST(suffixes); int totalChars = 0; for (String strip : seenStrips.keySet()) { totalChars += strip.length(); } stripData = new char[totalChars]; stripOffsets = new int[seenStrips.size() + 1]; int currentOffset = 0; int currentIndex = 0; for (String strip : seenStrips.keySet()) { stripOffsets[currentIndex++] = currentOffset; strip.getChars(0, strip.length(), stripData, currentOffset); currentOffset += strip.length(); } assert currentIndex == seenStrips.size(); stripOffsets[currentIndex] = currentOffset; } private FST<IntsRef> affixFST(TreeMap<String, List<Integer>> affixes) throws IOException { IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton(); Builder<IntsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, outputs); IntsRefBuilder scratch = new IntsRefBuilder(); for (Map.Entry<String, List<Integer>> entry : affixes.entrySet()) { Util.toUTF32(entry.getKey(), scratch); List<Integer> entries = entry.getValue(); IntsRef output = new IntsRef(entries.size()); for (Integer c : entries) { output.ints[output.length++] = c; } builder.add(scratch.get(), output); } return builder.finish(); } static String escapeDash(String re) { // we have to be careful, even though dash doesn't have a special meaning, // some dictionaries already escape it (e.g. pt_PT), so we don't want to nullify it StringBuilder escaped = new StringBuilder(); for (int i = 0; i < re.length(); i++) { char c = re.charAt(i); if (c == '-') { escaped.append("\\-"); } else { escaped.append(c); if (c == '\\' && i + 1 < re.length()) { escaped.append(re.charAt(i + 1)); i++; } } } return escaped.toString(); } /** * Parses a specific affix rule putting the result into the provided affix map * * @param affixes Map where the result of the parsing will be put * @param header Header line of the affix rule * @param reader BufferedReader to read the content of the rule from * @param conditionPattern {@link String#format(String, Object...)} pattern to be used to generate the condition regex * pattern * @param seenPatterns map from condition -> index of patterns, for deduplication. * @throws IOException Can be thrown while reading the rule */ private void parseAffix(TreeMap<String, List<Integer>> affixes, String header, LineNumberReader reader, String conditionPattern, Map<String, Integer> seenPatterns, Map<String, Integer> seenStrips) throws IOException, ParseException { BytesRefBuilder scratch = new BytesRefBuilder(); StringBuilder sb = new StringBuilder(); String args[] = header.split("\\s+"); boolean crossProduct = args[2].equals("Y"); boolean isSuffix = conditionPattern == SUFFIX_CONDITION_REGEX_PATTERN; int numLines = Integer.parseInt(args[3]); affixData = ArrayUtil.grow(affixData, (currentAffix << 3) + (numLines << 3)); ByteArrayDataOutput affixWriter = new ByteArrayDataOutput(affixData, currentAffix << 3, numLines << 3); for (int i = 0; i < numLines; i++) { assert affixWriter.getPosition() == currentAffix << 3; String line = reader.readLine(); String ruleArgs[] = line.split("\\s+"); // from the manpage: PFX flag stripping prefix [condition [morphological_fields...]] // condition is optional if (ruleArgs.length < 4) { throw new ParseException("The affix file contains a rule with less than four elements: " + line, reader.getLineNumber()); } char flag = flagParsingStrategy.parseFlag(ruleArgs[1]); String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2]; String affixArg = ruleArgs[3]; char appendFlags[] = null; // first: parse continuation classes out of affix int flagSep = affixArg.lastIndexOf('/'); if (flagSep != -1) { String flagPart = affixArg.substring(flagSep + 1); affixArg = affixArg.substring(0, flagSep); if (aliasCount > 0) { flagPart = getAliasValue(Integer.parseInt(flagPart)); } appendFlags = flagParsingStrategy.parseFlags(flagPart); Arrays.sort(appendFlags); twoStageAffix = true; } // zero affix -> empty string if ("0".equals(affixArg)) { affixArg = ""; } String condition = ruleArgs.length > 4 ? ruleArgs[4] : "."; // at least the gascon affix file has this issue if (condition.startsWith("[") && condition.indexOf(']') == -1) { condition = condition + "]"; } // "dash hasn't got special meaning" (we must escape it) if (condition.indexOf('-') >= 0) { condition = escapeDash(condition); } final String regex; if (".".equals(condition)) { regex = ".*"; // Zero condition is indicated by dot } else if (condition.equals(strip)) { regex = ".*"; // TODO: optimize this better: // if we remove 'strip' from condition, we don't have to append 'strip' to check it...! // but this is complicated... } else { regex = String.format(Locale.ROOT, conditionPattern, condition); } // deduplicate patterns Integer patternIndex = seenPatterns.get(regex); if (patternIndex == null) { patternIndex = patterns.size(); if (patternIndex > Short.MAX_VALUE) { throw new UnsupportedOperationException( "Too many patterns, please report this to dev@lucene.apache.org"); } seenPatterns.put(regex, patternIndex); CharacterRunAutomaton pattern = new CharacterRunAutomaton( new RegExp(regex, RegExp.NONE).toAutomaton()); patterns.add(pattern); } Integer stripOrd = seenStrips.get(strip); if (stripOrd == null) { stripOrd = seenStrips.size(); seenStrips.put(strip, stripOrd); if (stripOrd > Character.MAX_VALUE) { throw new UnsupportedOperationException( "Too many unique strips, please report this to dev@lucene.apache.org"); } } if (appendFlags == null) { appendFlags = NOFLAGS; } encodeFlags(scratch, appendFlags); int appendFlagsOrd = flagLookup.add(scratch.get()); if (appendFlagsOrd < 0) { // already exists in our hash appendFlagsOrd = (-appendFlagsOrd) - 1; } else if (appendFlagsOrd > Short.MAX_VALUE) { // this limit is probably flexible, but it's a good sanity check too throw new UnsupportedOperationException( "Too many unique append flags, please report this to dev@lucene.apache.org"); } affixWriter.writeShort((short) flag); affixWriter.writeShort((short) stripOrd.intValue()); // encode crossProduct into patternIndex int patternOrd = patternIndex.intValue() << 1 | (crossProduct ? 1 : 0); affixWriter.writeShort((short) patternOrd); affixWriter.writeShort((short) appendFlagsOrd); if (needsInputCleaning) { CharSequence cleaned = cleanInput(affixArg, sb); affixArg = cleaned.toString(); } if (isSuffix) { affixArg = new StringBuilder(affixArg).reverse().toString(); } List<Integer> list = affixes.get(affixArg); if (list == null) { list = new ArrayList<>(); affixes.put(affixArg, list); } list.add(currentAffix); currentAffix++; } } private FST<CharsRef> parseConversions(LineNumberReader reader, int num) throws IOException, ParseException { Map<String, String> mappings = new TreeMap<>(); for (int i = 0; i < num; i++) { String line = reader.readLine(); String parts[] = line.split("\\s+"); if (parts.length != 3) { throw new ParseException("invalid syntax: " + line, reader.getLineNumber()); } if (mappings.put(parts[1], parts[2]) != null) { throw new IllegalStateException("duplicate mapping specified for: " + parts[1]); } } Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton(); Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs); IntsRefBuilder scratchInts = new IntsRefBuilder(); for (Map.Entry<String, String> entry : mappings.entrySet()) { Util.toUTF16(entry.getKey(), scratchInts); builder.add(scratchInts.get(), new CharsRef(entry.getValue())); } return builder.finish(); } /** pattern accepts optional BOM + SET + any whitespace */ final static Pattern ENCODING_PATTERN = Pattern.compile("^(\u00EF\u00BB\u00BF)?SET\\s+"); /** * Parses the encoding specified in the affix file readable through the provided InputStream * * @param affix InputStream for reading the affix file * @return Encoding specified in the affix file * @throws IOException Can be thrown while reading from the InputStream * @throws ParseException Thrown if the first non-empty non-comment line read from the file does not adhere to the format {@code SET <encoding>} */ static String getDictionaryEncoding(InputStream affix) throws IOException, ParseException { final StringBuilder encoding = new StringBuilder(); for (;;) { encoding.setLength(0); int ch; while ((ch = affix.read()) >= 0) { if (ch == '\n') { break; } if (ch != '\r') { encoding.append((char) ch); } } if (encoding.length() == 0 || encoding.charAt(0) == '#' || // this test only at the end as ineffective but would allow lines only containing spaces: encoding.toString().trim().length() == 0) { if (ch < 0) { throw new ParseException("Unexpected end of affix file.", 0); } continue; } Matcher matcher = ENCODING_PATTERN.matcher(encoding); if (matcher.find()) { int last = matcher.end(); return encoding.substring(last).trim(); } } } static final Map<String, String> CHARSET_ALIASES; static { Map<String, String> m = new HashMap<>(); m.put("microsoft-cp1251", "windows-1251"); m.put("TIS620-2533", "TIS-620"); CHARSET_ALIASES = Collections.unmodifiableMap(m); } /** * Retrieves the CharsetDecoder for the given encoding. Note, This isn't perfect as I think ISCII-DEVANAGARI and * MICROSOFT-CP1251 etc are allowed... * * @param encoding Encoding to retrieve the CharsetDecoder for * @return CharSetDecoder for the given encoding */ private CharsetDecoder getJavaEncoding(String encoding) { if ("ISO8859-14".equals(encoding)) { return new ISO8859_14Decoder(); } String canon = CHARSET_ALIASES.get(encoding); if (canon != null) { encoding = canon; } Charset charset = Charset.forName(encoding); return charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE); } /** * Determines the appropriate {@link FlagParsingStrategy} based on the FLAG definition line taken from the affix file * * @param flagLine Line containing the flag information * @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG definition */ static FlagParsingStrategy getFlagParsingStrategy(String flagLine) { String parts[] = flagLine.split("\\s+"); if (parts.length != 2) { throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine); } String flagType = parts[1]; if (NUM_FLAG_TYPE.equals(flagType)) { return new NumFlagParsingStrategy(); } else if (UTF8_FLAG_TYPE.equals(flagType)) { return new SimpleFlagParsingStrategy(); } else if (LONG_FLAG_TYPE.equals(flagType)) { return new DoubleASCIIFlagParsingStrategy(); } throw new IllegalArgumentException("Unknown flag type: " + flagType); } final char FLAG_SEPARATOR = 0x1f; // flag separator after escaping final char MORPH_SEPARATOR = 0x1e; // separator for boundary of entry (may be followed by morph data) String unescapeEntry(String entry) { StringBuilder sb = new StringBuilder(); int end = morphBoundary(entry); for (int i = 0; i < end; i++) { char ch = entry.charAt(i); if (ch == '\\' && i + 1 < entry.length()) { sb.append(entry.charAt(i + 1)); i++; } else if (ch == '/') { sb.append(FLAG_SEPARATOR); } else if (ch == MORPH_SEPARATOR || ch == FLAG_SEPARATOR) { // BINARY EXECUTABLES EMBEDDED IN ZULU DICTIONARIES!!!!!!! } else { sb.append(ch); } } sb.append(MORPH_SEPARATOR); if (end < entry.length()) { for (int i = end; i < entry.length(); i++) { char c = entry.charAt(i); if (c == FLAG_SEPARATOR || c == MORPH_SEPARATOR) { // BINARY EXECUTABLES EMBEDDED IN ZULU DICTIONARIES!!!!!!! } else { sb.append(c); } } } return sb.toString(); } static int morphBoundary(String line) { int end = indexOfSpaceOrTab(line, 0); if (end == -1) { return line.length(); } while (end >= 0 && end < line.length()) { if (line.charAt(end) == '\t' || end + 3 < line.length() && Character.isLetter(line.charAt(end + 1)) && Character.isLetter(line.charAt(end + 2)) && line.charAt(end + 3) == ':') { break; } end = indexOfSpaceOrTab(line, end + 1); } if (end == -1) { return line.length(); } return end; } static int indexOfSpaceOrTab(String text, int start) { int pos1 = text.indexOf('\t', start); int pos2 = text.indexOf(' ', start); if (pos1 >= 0 && pos2 >= 0) { return Math.min(pos1, pos2); } else { return Math.max(pos1, pos2); } } /** * Reads the dictionary file through the provided InputStreams, building up the words map * * @param dictionaries InputStreams to read the dictionary file through * @param decoder CharsetDecoder used to decode the contents of the file * @throws IOException Can be thrown while reading from the file */ private void readDictionaryFiles(Directory tempDir, String tempFileNamePrefix, List<InputStream> dictionaries, CharsetDecoder decoder, Builder<IntsRef> words) throws IOException { BytesRefBuilder flagsScratch = new BytesRefBuilder(); IntsRefBuilder scratchInts = new IntsRefBuilder(); StringBuilder sb = new StringBuilder(); IndexOutput unsorted = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT); try (ByteSequencesWriter writer = new ByteSequencesWriter(unsorted)) { for (InputStream dictionary : dictionaries) { BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder)); String line = lines.readLine(); // first line is number of entries (approximately, sometimes) while ((line = lines.readLine()) != null) { // wild and unpredictable code comment rules if (line.isEmpty() || line.charAt(0) == '/' || line.charAt(0) == '#' || line.charAt(0) == '\t') { continue; } line = unescapeEntry(line); // if we havent seen any stem exceptions, try to parse one if (hasStemExceptions == false) { int morphStart = line.indexOf(MORPH_SEPARATOR); if (morphStart >= 0 && morphStart < line.length()) { hasStemExceptions = parseStemException(line.substring(morphStart + 1)) != null; } } if (needsInputCleaning) { int flagSep = line.indexOf(FLAG_SEPARATOR); if (flagSep == -1) { flagSep = line.indexOf(MORPH_SEPARATOR); } if (flagSep == -1) { CharSequence cleansed = cleanInput(line, sb); writer.write(cleansed.toString().getBytes(StandardCharsets.UTF_8)); } else { String text = line.substring(0, flagSep); CharSequence cleansed = cleanInput(text, sb); if (cleansed != sb) { sb.setLength(0); sb.append(cleansed); } sb.append(line.substring(flagSep)); writer.write(sb.toString().getBytes(StandardCharsets.UTF_8)); } } else { writer.write(line.getBytes(StandardCharsets.UTF_8)); } } } CodecUtil.writeFooter(unsorted); } OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, new Comparator<BytesRef>() { BytesRef scratch1 = new BytesRef(); BytesRef scratch2 = new BytesRef(); @Override public int compare(BytesRef o1, BytesRef o2) { scratch1.bytes = o1.bytes; scratch1.offset = o1.offset; scratch1.length = o1.length; for (int i = scratch1.length - 1; i >= 0; i--) { if (scratch1.bytes[scratch1.offset + i] == FLAG_SEPARATOR || scratch1.bytes[scratch1.offset + i] == MORPH_SEPARATOR) { scratch1.length = i; break; } } scratch2.bytes = o2.bytes; scratch2.offset = o2.offset; scratch2.length = o2.length; for (int i = scratch2.length - 1; i >= 0; i--) { if (scratch2.bytes[scratch2.offset + i] == FLAG_SEPARATOR || scratch2.bytes[scratch2.offset + i] == MORPH_SEPARATOR) { scratch2.length = i; break; } } int cmp = scratch1.compareTo(scratch2); if (cmp == 0) { // tie break on whole row return o1.compareTo(o2); } else { return cmp; } } }); String sorted; boolean success = false; try { sorted = sorter.sort(unsorted.getName()); success = true; } finally { if (success) { tempDir.deleteFile(unsorted.getName()); } else { IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName()); } } boolean success2 = false; try (ByteSequencesReader reader = new ByteSequencesReader( tempDir.openChecksumInput(sorted, IOContext.READONCE), sorted)) { // TODO: the flags themselves can be double-chars (long) or also numeric // either way the trick is to encode them as char... but they must be parsed differently String currentEntry = null; IntsRefBuilder currentOrds = new IntsRefBuilder(); while (true) { BytesRef scratch = reader.next(); if (scratch == null) { break; } String line = scratch.utf8ToString(); String entry; char wordForm[]; int end; int flagSep = line.indexOf(FLAG_SEPARATOR); if (flagSep == -1) { wordForm = NOFLAGS; end = line.indexOf(MORPH_SEPARATOR); entry = line.substring(0, end); } else { end = line.indexOf(MORPH_SEPARATOR); String flagPart = line.substring(flagSep + 1, end); if (aliasCount > 0) { flagPart = getAliasValue(Integer.parseInt(flagPart)); } wordForm = flagParsingStrategy.parseFlags(flagPart); Arrays.sort(wordForm); entry = line.substring(0, flagSep); } // we possibly have morphological data int stemExceptionID = 0; if (hasStemExceptions && end + 1 < line.length()) { String stemException = parseStemException(line.substring(end + 1)); if (stemException != null) { stemExceptions = ArrayUtil.grow(stemExceptions, stemExceptionCount + 1); stemExceptionID = stemExceptionCount + 1; // we use '0' to indicate no exception for the form stemExceptions[stemExceptionCount++] = stemException; } } int cmp = currentEntry == null ? 1 : entry.compareTo(currentEntry); if (cmp < 0) { throw new IllegalArgumentException("out of order: " + entry + " < " + currentEntry); } else { encodeFlags(flagsScratch, wordForm); int ord = flagLookup.add(flagsScratch.get()); if (ord < 0) { // already exists in our hash ord = (-ord) - 1; } // finalize current entry, and switch "current" if necessary if (cmp > 0 && currentEntry != null) { Util.toUTF32(currentEntry, scratchInts); words.add(scratchInts.get(), currentOrds.get()); } // swap current if (cmp > 0 || currentEntry == null) { currentEntry = entry; currentOrds = new IntsRefBuilder(); // must be this way } if (hasStemExceptions) { currentOrds.append(ord); currentOrds.append(stemExceptionID); } else { currentOrds.append(ord); } } } // finalize last entry Util.toUTF32(currentEntry, scratchInts); words.add(scratchInts.get(), currentOrds.get()); success2 = true; } finally { if (success2) { tempDir.deleteFile(sorted); } else { IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted); } } } static char[] decodeFlags(BytesRef b) { if (b.length == 0) { return CharsRef.EMPTY_CHARS; } int len = b.length >>> 1; char flags[] = new char[len]; int upto = 0; int end = b.offset + b.length; for (int i = b.offset; i < end; i += 2) { flags[upto++] = (char) ((b.bytes[i] << 8) | (b.bytes[i + 1] & 0xff)); } return flags; } static void encodeFlags(BytesRefBuilder b, char flags[]) { int len = flags.length << 1; b.grow(len); b.clear(); for (int i = 0; i < flags.length; i++) { int flag = flags[i]; b.append((byte) ((flag >> 8) & 0xff)); b.append((byte) (flag & 0xff)); } } private void parseAlias(String line) { String ruleArgs[] = line.split("\\s+"); if (aliases == null) { //first line should be the aliases count final int count = Integer.parseInt(ruleArgs[1]); aliases = new String[count]; } else { // an alias can map to no flags String aliasValue = ruleArgs.length == 1 ? "" : ruleArgs[1]; aliases[aliasCount++] = aliasValue; } } private String getAliasValue(int id) { try { return aliases[id - 1]; } catch (IndexOutOfBoundsException ex) { throw new IllegalArgumentException("Bad flag alias number:" + id, ex); } } String getStemException(int id) { return stemExceptions[id - 1]; } private void parseMorphAlias(String line) { if (morphAliases == null) { //first line should be the aliases count final int count = Integer.parseInt(line.substring(3)); morphAliases = new String[count]; } else { String arg = line.substring(2); // leave the space morphAliases[morphAliasCount++] = arg; } } private String parseStemException(String morphData) { // first see if it's an alias if (morphAliasCount > 0) { try { int alias = Integer.parseInt(morphData.trim()); morphData = morphAliases[alias - 1]; } catch (NumberFormatException e) { // fine } } // try to parse morph entry int index = morphData.indexOf(" st:"); if (index < 0) { index = morphData.indexOf("\tst:"); } if (index >= 0) { int endIndex = indexOfSpaceOrTab(morphData, index + 1); if (endIndex < 0) { endIndex = morphData.length(); } return morphData.substring(index + 4, endIndex); } return null; } /** * Abstraction of the process of parsing flags taken from the affix and dic files */ static abstract class FlagParsingStrategy { /** * Parses the given String into a single flag * * @param rawFlag String to parse into a flag * @return Parsed flag */ char parseFlag(String rawFlag) { char flags[] = parseFlags(rawFlag); if (flags.length != 1) { throw new IllegalArgumentException("expected only one flag, got: " + rawFlag); } return flags[0]; } /** * Parses the given String into multiple flags * * @param rawFlags String to parse into flags * @return Parsed flags */ abstract char[] parseFlags(String rawFlags); } /** * Simple implementation of {@link FlagParsingStrategy} that treats the chars in each String as a individual flags. * Can be used with both the ASCII and UTF-8 flag types. */ private static class SimpleFlagParsingStrategy extends FlagParsingStrategy { @Override public char[] parseFlags(String rawFlags) { return rawFlags.toCharArray(); } } /** * Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its numerical form. In the case * of multiple flags, each number is separated by a comma. */ private static class NumFlagParsingStrategy extends FlagParsingStrategy { @Override public char[] parseFlags(String rawFlags) { String[] rawFlagParts = rawFlags.trim().split(","); char[] flags = new char[rawFlagParts.length]; int upto = 0; for (int i = 0; i < rawFlagParts.length; i++) { // note, removing the trailing X/leading I for nepali... what is the rule here?! String replacement = rawFlagParts[i].replaceAll("[^0-9]", ""); // note, ignoring empty flags (this happens in danish, for example) if (replacement.isEmpty()) { continue; } flags[upto++] = (char) Integer.parseInt(replacement); } if (upto < flags.length) { flags = ArrayUtil.copyOfSubArray(flags, 0, upto); } return flags; } } /** * Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded as two ASCII characters whose codes * must be combined into a single character. */ private static class DoubleASCIIFlagParsingStrategy extends FlagParsingStrategy { @Override public char[] parseFlags(String rawFlags) { if (rawFlags.length() == 0) { return new char[0]; } StringBuilder builder = new StringBuilder(); if (rawFlags.length() % 2 == 1) { throw new IllegalArgumentException( "Invalid flags (should be even number of characters): " + rawFlags); } for (int i = 0; i < rawFlags.length(); i += 2) { char f1 = rawFlags.charAt(i); char f2 = rawFlags.charAt(i + 1); if (f1 >= 256 || f2 >= 256) { throw new IllegalArgumentException( "Invalid flags (LONG flags must be double ASCII): " + rawFlags); } char combined = (char) (f1 << 8 | f2); builder.append(combined); } char flags[] = new char[builder.length()]; builder.getChars(0, builder.length(), flags, 0); return flags; } } static boolean hasFlag(char flags[], char flag) { return Arrays.binarySearch(flags, flag) >= 0; } CharSequence cleanInput(CharSequence input, StringBuilder reuse) { reuse.setLength(0); for (int i = 0; i < input.length(); i++) { char ch = input.charAt(i); if (ignore != null && Arrays.binarySearch(ignore, ch) >= 0) { continue; } if (ignoreCase && iconv == null) { // if we have no input conversion mappings, do this on-the-fly ch = caseFold(ch); } reuse.append(ch); } if (iconv != null) { try { applyMappings(iconv, reuse); } catch (IOException bogus) { throw new RuntimeException(bogus); } if (ignoreCase) { for (int i = 0; i < reuse.length(); i++) { reuse.setCharAt(i, caseFold(reuse.charAt(i))); } } } return reuse; } /** folds single character (according to LANG if present) */ char caseFold(char c) { if (alternateCasing) { if (c == 'I') { return ''; } else if (c == '') { return 'i'; } else { return Character.toLowerCase(c); } } else { return Character.toLowerCase(c); } } // TODO: this could be more efficient! static void applyMappings(FST<CharsRef> fst, StringBuilder sb) throws IOException { final FST.BytesReader bytesReader = fst.getBytesReader(); final FST.Arc<CharsRef> firstArc = fst.getFirstArc(new FST.Arc<CharsRef>()); final CharsRef NO_OUTPUT = fst.outputs.getNoOutput(); // temporary stuff final FST.Arc<CharsRef> arc = new FST.Arc<>(); int longestMatch; CharsRef longestOutput; for (int i = 0; i < sb.length(); i++) { arc.copyFrom(firstArc); CharsRef output = NO_OUTPUT; longestMatch = -1; longestOutput = null; for (int j = i; j < sb.length(); j++) { char ch = sb.charAt(j); if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) { break; } else { output = fst.outputs.add(output, arc.output); } if (arc.isFinal()) { longestOutput = fst.outputs.add(output, arc.nextFinalOutput); longestMatch = j; } } if (longestMatch >= 0) { sb.delete(i, longestMatch + 1); sb.insert(i, longestOutput); i += (longestOutput.length - 1); } } } /** Returns true if this dictionary was constructed with the {@code ignoreCase} option */ public boolean getIgnoreCase() { return ignoreCase; } private static Path DEFAULT_TEMP_DIR; /** Used by test framework */ public static void setDefaultTempDir(Path tempDir) { DEFAULT_TEMP_DIR = tempDir; } /** * Returns the default temporary directory. By default, java.io.tmpdir. If not accessible * or not available, an IOException is thrown */ synchronized static Path getDefaultTempDir() throws IOException { if (DEFAULT_TEMP_DIR == null) { // Lazy init String tempDirPath = System.getProperty("java.io.tmpdir"); if (tempDirPath == null) { throw new IOException("Java has no temporary folder property (java.io.tmpdir)?"); } Path tempDirectory = Paths.get(tempDirPath); if (Files.isWritable(tempDirectory) == false) { throw new IOException( "Java's temporary folder not present or writeable?: " + tempDirectory.toAbsolutePath()); } DEFAULT_TEMP_DIR = tempDirectory; } return DEFAULT_TEMP_DIR; } }