Example usage for org.apache.lucene.search.spell SuggestMode SUGGEST_WHEN_NOT_IN_INDEX

Introduction

In this page you can find the example usage for org.apache.lucene.search.spell SuggestMode SUGGEST_WHEN_NOT_IN_INDEX.

Prototype

SuggestMode SUGGEST_WHEN_NOT_IN_INDEX

To view the source code for org.apache.lucene.search.spell SuggestMode SUGGEST_WHEN_NOT_IN_INDEX.

Click Source Link

Document

Generate suggestions only for terms not in the index (default)

Usage

From source file:org.apache.solr.handler.component.SpellCheckComponent.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public void process(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    if (!params.getBool(COMPONENT_NAME, false) || spellCheckers.isEmpty()) {
        return;//  ww  w .  j  a  va2  s. c  o  m
    }
    boolean shardRequest = "true".equals(params.get(ShardParams.IS_SHARD));
    String q = params.get(SPELLCHECK_Q);
    SolrSpellChecker spellChecker = getSpellChecker(params);
    Collection<Token> tokens = null;

    if (q != null) {
        //we have a spell check param, tokenize it with the query analyzer applicable for this spellchecker
        tokens = getTokens(q, spellChecker.getQueryAnalyzer());
    } else {
        q = rb.getQueryString();
        if (q == null) {
            q = params.get(CommonParams.Q);
        }
        tokens = queryConverter.convert(q);
    }
    if (tokens != null && tokens.isEmpty() == false) {
        if (spellChecker != null) {
            int count = params.getInt(SPELLCHECK_COUNT, 1);
            boolean onlyMorePopular = params.getBool(SPELLCHECK_ONLY_MORE_POPULAR, DEFAULT_ONLY_MORE_POPULAR);
            boolean extendedResults = params.getBool(SPELLCHECK_EXTENDED_RESULTS, false);
            boolean collate = params.getBool(SPELLCHECK_COLLATE, false);
            float accuracy = params.getFloat(SPELLCHECK_ACCURACY, Float.MIN_VALUE);
            Integer alternativeTermCount = params.getInt(SpellingParams.SPELLCHECK_ALTERNATIVE_TERM_COUNT);
            Integer maxResultsForSuggest = params.getInt(SpellingParams.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST);
            ModifiableSolrParams customParams = new ModifiableSolrParams();
            for (String checkerName : getDictionaryNames(params)) {
                customParams.add(getCustomParams(checkerName, params));
            }

            Integer hitsInteger = (Integer) rb.rsp.getToLog().get("hits");
            long hits = 0;
            if (hitsInteger == null) {
                hits = rb.getNumberDocumentsFound();
            } else {
                hits = hitsInteger.longValue();
            }
            SpellingResult spellingResult = null;
            if (maxResultsForSuggest == null || hits <= maxResultsForSuggest) {
                SuggestMode suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
                if (onlyMorePopular) {
                    suggestMode = SuggestMode.SUGGEST_MORE_POPULAR;
                } else if (alternativeTermCount != null) {
                    suggestMode = SuggestMode.SUGGEST_ALWAYS;
                }

                IndexReader reader = rb.req.getSearcher().getIndexReader();
                SpellingOptions options = new SpellingOptions(tokens, reader, count, alternativeTermCount,
                        suggestMode, extendedResults, accuracy, customParams);
                spellingResult = spellChecker.getSuggestions(options);
            } else {
                spellingResult = new SpellingResult();
            }
            boolean isCorrectlySpelled = hits > (maxResultsForSuggest == null ? 0 : maxResultsForSuggest);
            NamedList suggestions = toNamedList(shardRequest, spellingResult, q, extendedResults, collate,
                    isCorrectlySpelled);
            if (collate) {
                addCollationsToResponse(params, spellingResult, rb, q, suggestions,
                        spellChecker.isSuggestionsMayOverlap());
            }
            NamedList response = new SimpleOrderedMap();
            response.add("suggestions", suggestions);
            rb.rsp.add("spellcheck", response);

        } else {
            throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Specified dictionaries do not exist: "
                    + getDictionaryNameAsSingleString(getDictionaryNames(params)));
        }
    }
}

---

From source file:org.apache.solr.spelling.IndexBasedSpellCheckerTest.java

License:Apache License

@Test
public void testExtendedResults() throws Exception {
    IndexBasedSpellChecker checker = new IndexBasedSpellChecker();
    NamedList spellchecker = new NamedList();
    spellchecker.add("classname", IndexBasedSpellChecker.class.getName());

    File indexDir = new File(TEMP_DIR, "spellingIdx" + new Date().getTime());
    indexDir.mkdirs();/*from   w ww  .  j  a  v a 2s.  com*/
    spellchecker.add(AbstractLuceneSpellChecker.INDEX_DIR, indexDir.getAbsolutePath());
    spellchecker.add(AbstractLuceneSpellChecker.FIELD, "title");
    spellchecker.add(AbstractLuceneSpellChecker.SPELLCHECKER_ARG_NAME, spellchecker);
    SolrCore core = h.getCore();
    String dictName = checker.init(spellchecker, core);
    assertTrue(dictName + " is not equal to " + SolrSpellChecker.DEFAULT_DICTIONARY_NAME,
            dictName.equals(SolrSpellChecker.DEFAULT_DICTIONARY_NAME) == true);
    RefCounted<SolrIndexSearcher> holder = core.getSearcher();
    SolrIndexSearcher searcher = holder.get();
    try {
        checker.build(core, searcher);

        IndexReader reader = searcher.getIndexReader();
        Collection<Token> tokens = queryConverter.convert("documemt");
        SpellingOptions spellOpts = new SpellingOptions(tokens, reader, 1,
                SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);
        SpellingResult result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        //should be lowercased, b/c we are using a lowercasing analyzer
        Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("documemt is null and it shouldn't be", suggestions != null);
        assertTrue("documemt Size: " + suggestions.size() + " is not: " + 1, suggestions.size() == 1);
        Map.Entry<String, Integer> entry = suggestions.entrySet().iterator().next();
        assertTrue(entry.getKey() + " is not equal to " + "document",
                entry.getKey().equals("document") == true);
        assertTrue(entry.getValue() + " does not equal: " + 2, entry.getValue() == 2);

        //test something not in the spell checker
        spellOpts.tokens = queryConverter.convert("super");
        result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("suggestions size should be 0", suggestions.size() == 0);

        spellOpts.tokens = queryConverter.convert("document");
        result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("suggestions is not null and it should be", suggestions == null);
    } finally {
        holder.decref();
    }
}

---

From source file:org.apache.solr.spelling.IndexBasedSpellCheckerTest.java

License:Apache License

@Test
public void testAlternateLocation() throws Exception {
    String[] ALT_DOCS = new String[] { "jumpin jack flash", "Sargent Peppers Lonely Hearts Club Band",
            "Born to Run", "Thunder Road", "Londons Burning", "A Horse with No Name", "Sweet Caroline" };

    IndexBasedSpellChecker checker = new IndexBasedSpellChecker();
    NamedList spellchecker = new NamedList();
    spellchecker.add("classname", IndexBasedSpellChecker.class.getName());

    File indexDir = new File(TEMP_DIR, "spellingIdx" + new Date().getTime());
    //create a standalone index
    File altIndexDir = new File(TEMP_DIR, "alternateIdx" + new Date().getTime());
    Directory dir = newFSDirectory(altIndexDir);
    IndexWriter iw = new IndexWriter(dir,
            new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
    for (int i = 0; i < ALT_DOCS.length; i++) {
        Document doc = new Document();
        doc.add(new TextField("title", ALT_DOCS[i], Field.Store.YES));
        iw.addDocument(doc);//from   w w  w .j  av a 2 s  . com
    }
    iw.forceMerge(1);
    iw.close();
    dir.close();
    indexDir.mkdirs();
    spellchecker.add(AbstractLuceneSpellChecker.INDEX_DIR, indexDir.getAbsolutePath());
    spellchecker.add(AbstractLuceneSpellChecker.LOCATION, altIndexDir.getAbsolutePath());
    spellchecker.add(AbstractLuceneSpellChecker.FIELD, "title");
    spellchecker.add(AbstractLuceneSpellChecker.SPELLCHECKER_ARG_NAME, spellchecker);
    SolrCore core = h.getCore();
    String dictName = checker.init(spellchecker, core);
    assertTrue(dictName + " is not equal to " + SolrSpellChecker.DEFAULT_DICTIONARY_NAME,
            dictName.equals(SolrSpellChecker.DEFAULT_DICTIONARY_NAME) == true);
    RefCounted<SolrIndexSearcher> holder = core.getSearcher();
    SolrIndexSearcher searcher = holder.get();
    try {
        checker.build(core, searcher);

        IndexReader reader = searcher.getIndexReader();
        Collection<Token> tokens = queryConverter.convert("flesh");
        SpellingOptions spellOpts = new SpellingOptions(tokens, reader, 1,
                SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);
        SpellingResult result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        //should be lowercased, b/c we are using a lowercasing analyzer
        Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("flesh is null and it shouldn't be", suggestions != null);
        assertTrue("flesh Size: " + suggestions.size() + " is not: " + 1, suggestions.size() == 1);
        Map.Entry<String, Integer> entry = suggestions.entrySet().iterator().next();
        assertTrue(entry.getKey() + " is not equal to " + "flash", entry.getKey().equals("flash") == true);
        assertTrue(entry.getValue() + " does not equal: " + 1, entry.getValue() == 1);

        //test something not in the spell checker
        spellOpts.tokens = queryConverter.convert("super");
        result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("suggestions size should be 0", suggestions.size() == 0);

        spellOpts.tokens = queryConverter.convert("Caroline");
        result = checker.getSuggestions(spellOpts);
        assertTrue("result is null and it shouldn't be", result != null);
        suggestions = result.get(spellOpts.tokens.iterator().next());
        assertTrue("suggestions is not null and it should be", suggestions == null);
    } finally {
        holder.decref();
    }
}

---

From source file:org.dice.solrenhancements.spellchecker.DiceSpellCheckComponent.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public void process(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    if (!params.getBool(COMPONENT_NAME, false) || spellCheckers.isEmpty()) {
        return;//from   w  ww  .  java2 s.  c  o m
    }
    boolean shardRequest = "true".equals(params.get(ShardParams.IS_SHARD));
    String q = params.get(SPELLCHECK_Q);
    SolrSpellChecker spellChecker = getSpellChecker(params);
    Collection<Token> tokens = null;

    if (q == null) {
        // enforce useage of the spellcheck.q parameter - i.e. a query we can tokenize with a regular tokenizer and not
        // a solr query for the spell checking. Useage of the SolrQueryConverter is buggy and breaks frequently
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The spellcheck.q parameter is required.");
    } else {
        //we have a spell check param, tokenize it with the query analyzer applicable for this spellchecker
        tokens = getTokens(q, spellChecker.getQueryAnalyzer());
    }
    if (tokens != null && tokens.isEmpty() == false) {
        if (spellChecker != null) {
            int count = params.getInt(SPELLCHECK_COUNT, 1);
            boolean onlyMorePopular = params.getBool(SPELLCHECK_ONLY_MORE_POPULAR, DEFAULT_ONLY_MORE_POPULAR);
            boolean extendedResults = params.getBool(SPELLCHECK_EXTENDED_RESULTS, false);
            boolean collate = params.getBool(SPELLCHECK_COLLATE, false);
            float accuracy = params.getFloat(SPELLCHECK_ACCURACY, Float.MIN_VALUE);
            Integer alternativeTermCount = params.getInt(SpellingParams.SPELLCHECK_ALTERNATIVE_TERM_COUNT);
            Integer maxResultsForSuggest = params.getInt(SpellingParams.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST);
            ModifiableSolrParams customParams = new ModifiableSolrParams();
            for (String checkerName : getDictionaryNames(params)) {
                customParams.add(getCustomParams(checkerName, params));
            }

            Integer hitsInteger = (Integer) rb.rsp.getToLog().get("hits");
            long hits = 0;
            if (hitsInteger == null) {
                hits = rb.getNumberDocumentsFound();
            } else {
                hits = hitsInteger.longValue();
            }
            SpellingResult spellingResult = null;
            if (maxResultsForSuggest == null || hits <= maxResultsForSuggest) {
                SuggestMode suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
                if (onlyMorePopular) {
                    suggestMode = SuggestMode.SUGGEST_MORE_POPULAR;
                } else if (alternativeTermCount != null) {
                    suggestMode = SuggestMode.SUGGEST_ALWAYS;
                }

                IndexReader reader = rb.req.getSearcher().getIndexReader();
                SpellingOptions options = new SpellingOptions(tokens, reader, count, alternativeTermCount,
                        suggestMode, extendedResults, accuracy, customParams);
                spellingResult = spellChecker.getSuggestions(options);
            } else {
                spellingResult = new SpellingResult();
            }
            boolean isCorrectlySpelled = hits > (maxResultsForSuggest == null ? 0 : maxResultsForSuggest);
            NamedList suggestions = toNamedList(shardRequest, spellingResult, q, extendedResults, collate,
                    isCorrectlySpelled);
            if (collate) {
                ModifiableSolrParams modParams = new ModifiableSolrParams(params);
                // SH: having both spellcheck.q and q set screws up collations for some queries, such as "java develope"
                modParams.remove(CommonParams.Q);

                //SH: Note that the collator runs a query against the DF specified field. Ideally it should
                //run the query against the spellchecker field but that's inaccessible here
                addCollationsToResponse(modParams, spellingResult, rb, q, suggestions,
                        spellChecker.isSuggestionsMayOverlap());
            }
            NamedList response = new SimpleOrderedMap();
            response.add("suggestions", suggestions);
            rb.rsp.add("spellcheck", response);

        } else {
            throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Specified dictionaries do not exist: "
                    + getDictionaryNameAsSingleString(getDictionaryNames(params)));
        }
    }
}

---

From source file:org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder.java

License:Apache License

private static SuggestMode resolveSuggestMode(String suggestMode) {
    suggestMode = suggestMode.toLowerCase(Locale.US);
    if ("missing".equals(suggestMode)) {
        return SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
    } else if ("popular".equals(suggestMode)) {
        return SuggestMode.SUGGEST_MORE_POPULAR;
    } else if ("always".equals(suggestMode)) {
        return SuggestMode.SUGGEST_ALWAYS;
    } else {//from  w w w . ja  v  a2  s .co m
        throw new IllegalArgumentException("Illegal suggest mode " + suggestMode);
    }
}

---

From source file:org.elasticsearch.search.suggest.SuggestUtils.java

License:Apache License

public static SuggestMode resolveSuggestMode(String suggestMode) {
    suggestMode = suggestMode.toLowerCase(Locale.US);
    if ("missing".equals(suggestMode)) {
        return SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
    } else if ("popular".equals(suggestMode)) {
        return SuggestMode.SUGGEST_MORE_POPULAR;
    } else if ("always".equals(suggestMode)) {
        return SuggestMode.SUGGEST_ALWAYS;
    } else {//from  ww  w . ja  v a 2  s.  co m
        throw new ElasticsearchIllegalArgumentException("Illegal suggest mode " + suggestMode);
    }
}

---