Example usage for org.apache.solr.analysis TokenizerChain TokenizerChain

Introduction

In this page you can find the example usage for org.apache.solr.analysis TokenizerChain TokenizerChain.

Prototype

public TokenizerChain(TokenizerFactory tokenizer, TokenFilterFactory[] filters) 

Source Link

Document

Creates a new TokenizerChain w/o any CharFilterFactories.

Usage

From source file:com.github.healthonnet.search.SynonymExpandingExtendedDismaxQParserPlugin.java

License:Apache License

private void parseConfig(Map<String, Analyzer> analyzers, String argName) {
    try {//from w w w .  ja  v a2 s.c o m

        Object xmlAnalyzers = args.get(argName);

        if (xmlAnalyzers != null && xmlAnalyzers instanceof NamedList) {
            NamedList<?> AnalyzersList = (NamedList<?>) xmlAnalyzers;
            for (Entry<String, ?> entry : AnalyzersList) {
                String analyzerName = entry.getKey();
                if (!(entry.getValue() instanceof NamedList)) {
                    continue;
                }
                NamedList<?> analyzerAsNamedList = (NamedList<?>) entry.getValue();

                TokenizerFactory tokenizerFactory = null;
                TokenFilterFactory filterFactory;
                List<TokenFilterFactory> filterFactories = new LinkedList<>();

                for (Entry<String, ?> analyzerEntry : analyzerAsNamedList) {
                    String key = analyzerEntry.getKey();
                    if (!(entry.getValue() instanceof NamedList)) {
                        continue;
                    }
                    Map<String, String> params = convertNamedListToMap((NamedList<?>) analyzerEntry.getValue());

                    String className = params.get("class");
                    if (className == null) {
                        continue;
                    }

                    params.put("luceneMatchVersion", luceneMatchVersion.toString());

                    if (key.equals("tokenizer")) {
                        try {
                            tokenizerFactory = TokenizerFactory.forName(className, params);
                        } catch (IllegalArgumentException iae) {
                            if (!className.contains(".")) {
                                iae.printStackTrace();
                            }
                            // Now try by classname instead of SPI keyword
                            tokenizerFactory = loader.newInstance(className, TokenizerFactory.class,
                                    new String[] {}, new Class[] { Map.class }, new Object[] { params });
                        }
                        if (tokenizerFactory instanceof ResourceLoaderAware) {
                            ((ResourceLoaderAware) tokenizerFactory).inform(loader);
                        }
                    } else if (key.equals("filter")) {
                        try {
                            filterFactory = TokenFilterFactory.forName(className, params);
                        } catch (IllegalArgumentException iae) {
                            if (!className.contains(".")) {
                                iae.printStackTrace();
                            }
                            // Now try by classname instead of SPI keyword
                            filterFactory = loader.newInstance(className, TokenFilterFactory.class,
                                    new String[] {}, new Class[] { Map.class }, new Object[] { params });
                        }
                        if (filterFactory instanceof ResourceLoaderAware) {
                            ((ResourceLoaderAware) filterFactory).inform(loader);
                        }
                        filterFactories.add(filterFactory);
                    }
                }
                if (tokenizerFactory == null) {
                    throw new SolrException(ErrorCode.SERVER_ERROR,
                            "tokenizer must not be null for analyzer: " + analyzerName);
                } else if (filterFactories.isEmpty()) {
                    throw new SolrException(ErrorCode.SERVER_ERROR,
                            "filter factories must be defined for analyzer: " + analyzerName);
                }

                TokenizerChain analyzer = new TokenizerChain(tokenizerFactory,
                        filterFactories.toArray(new TokenFilterFactory[filterFactories.size()]));

                analyzers.put(analyzerName, analyzer);
            }
        }
    } catch (IOException e) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to create parser. Check your config.", e);
    }
}

---

From source file:NomusSolrPlugins.NomusDismaxQParserPlugin.java

License:Apache License

public TokenStream tokenStream(String fieldName, Reader reader) {
    if (!removeStopFilter) {
        return queryAnalyzer.tokenStream(fieldName, reader);
    }/*from   ww w  .  jav  a  2  s  .  c  o m*/

    Analyzer a = map.get(fieldName);
    if (a != null) {
        return a.tokenStream(fieldName, reader);
    }

    FieldType ft = parser.getReq().getSchema().getFieldType(fieldName);
    Analyzer qa = ft.getQueryAnalyzer();
    if (!(qa instanceof TokenizerChain)) {
        map.put(fieldName, qa);
        return qa.tokenStream(fieldName, reader);
    }
    TokenizerChain tcq = (TokenizerChain) qa;
    Analyzer ia = ft.getAnalyzer();
    if (ia == qa || !(ia instanceof TokenizerChain)) {
        map.put(fieldName, qa);
        return qa.tokenStream(fieldName, reader);
    }
    TokenizerChain tci = (TokenizerChain) ia;

    // make sure that there isn't a stop filter in the indexer
    for (TokenFilterFactory tf : tci.getTokenFilterFactories()) {
        if (tf instanceof StopFilterFactory) {
            map.put(fieldName, qa);
            return qa.tokenStream(fieldName, reader);
        }
    }

    // now if there is a stop filter in the query analyzer, remove it
    int stopIdx = -1;
    TokenFilterFactory[] facs = tcq.getTokenFilterFactories();

    for (int i = 0; i < facs.length; i++) {
        TokenFilterFactory tf = facs[i];
        if (tf instanceof StopFilterFactory) {
            stopIdx = i;
            break;
        }
    }

    if (stopIdx == -1) {
        // no stop filter exists
        map.put(fieldName, qa);
        return qa.tokenStream(fieldName, reader);
    }

    TokenFilterFactory[] newtf = new TokenFilterFactory[facs.length - 1];
    for (int i = 0, j = 0; i < facs.length; i++) {
        if (i == stopIdx)
            continue;
        newtf[j++] = facs[i];
    }

    TokenizerChain newa = new TokenizerChain(tcq.getTokenizerFactory(), newtf);
    newa.setPositionIncrementGap(tcq.getPositionIncrementGap(fieldName));

    map.put(fieldName, newa);
    return newa.tokenStream(fieldName, reader);
}

---

From source file:org.sindice.siren.solr.qparser.tuple.TupleQParser.java

License:Open Source License

/**
 * Initialise the Tuple query analyzer//  w  ww.  j  a v a 2 s . co  m
 */
private Analyzer initAnalyzer() {
    return new TokenizerChain(getTupleTokenizerFactory(), new TokenFilterFactory[0]);
}

---