Example usage for org.apache.lucene.queries.mlt MoreLikeThis setMinDocFreq

Introduction

In this page you can find the example usage for org.apache.lucene.queries.mlt MoreLikeThis setMinDocFreq.

Prototype

public void setMinDocFreq(int minDocFreq) 

Source Link

Document

Sets the frequency at which words will be ignored which do not occur in at least this many docs.

Usage

From source file:org.cee.store.lucene.LuceneArticleStore.java

License:Apache License

private Query createRelatedArticlesQuery(List<EntityKey> sites, ArticleKey reference, IndexSearcher searcher,
        String language) throws IOException {
    Query articleQuery = createArticleQuery(reference);
    TopDocs topDocs = searcher.search(articleQuery, 1);
    if (topDocs.totalHits == 0) {
        return new BooleanQuery(true);
    }/*from   w  ww  .  j a  va2s  .  co  m*/
    MoreLikeThis mlt = new MoreLikeThis(searcher.getIndexReader());
    mlt.setFieldNames(LuceneConstants.ARTICLE_RELATED_SEARCH_FIELDS);
    mlt.setMaxQueryTerms(20);
    mlt.setBoost(true);
    mlt.setMinTermFreq(0);
    mlt.setMinDocFreq(0);
    Query relatedQuery = boostRelatedQuery(mlt.like(topDocs.scoreDocs[0].doc));

    BooleanQuery query = new BooleanQuery();
    query.add(new BooleanClause(relatedQuery, Occur.MUST));
    query.add(new BooleanClause(createQueryArticlesOfSites(sites), Occur.MUST));
    return query;
}

---

From source file:org.elasticsearch.common.lucene.search.morelikethis.XMoreLikeThisTests.java

License:Apache License

@Test
public void testTopN() throws Exception {
    int numDocs = 100;
    int topN = 25;

    // add series of docs with terms of decreasing df
    Directory dir = newDirectory();// w ww  .  j  a va 2  s.  c  om
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    for (int i = 0; i < numDocs; i++) {
        addDoc(writer, generateStrSeq(0, i + 1));
    }
    IndexReader reader = writer.getReader();
    writer.close();

    // setup MLT query
    MoreLikeThis mlt = new MoreLikeThis(reader);
    mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
    mlt.setMaxQueryTerms(topN);
    mlt.setMinDocFreq(1);
    mlt.setMinTermFreq(1);
    mlt.setMinWordLen(1);
    mlt.setFieldNames(new String[] { "text" });

    // perform MLT query
    String likeText = "";
    for (String text : generateStrSeq(0, numDocs)) {
        likeText += text + " ";
    }
    BooleanQuery query = (BooleanQuery) mlt.like("text", new StringReader(likeText));

    // check best terms are topN of highest idf
    List<BooleanClause> clauses = query.clauses();
    assertEquals("Expected" + topN + "clauses only!", topN, clauses.size());

    Term[] expectedTerms = new Term[topN];
    int idx = 0;
    for (String text : generateStrSeq(numDocs - topN, topN)) {
        expectedTerms[idx++] = new Term("text", text);
    }
    for (BooleanClause clause : clauses) {
        Term term = ((TermQuery) clause.getQuery()).getTerm();
        assertTrue(Arrays.asList(expectedTerms).contains(term));
    }

    // clean up
    reader.close();
    dir.close();
}

---

From source file:org.elasticsearch.common.lucene.search.MoreLikeThisQuery.java

License:Apache License

@Override
public Query rewrite(IndexReader reader) throws IOException {
    MoreLikeThis mlt = new MoreLikeThis(reader, similarity == null ? new DefaultSimilarity() : similarity);

    mlt.setFieldNames(moreLikeFields);//w  w w .ja v a 2 s. com
    mlt.setAnalyzer(analyzer);
    mlt.setMinTermFreq(minTermFrequency);
    mlt.setMinDocFreq(minDocFreq);
    mlt.setMaxDocFreq(maxDocFreq);
    mlt.setMaxQueryTerms(maxQueryTerms);
    mlt.setMinWordLen(minWordLen);
    mlt.setMaxWordLen(maxWordLen);
    mlt.setStopWords(stopWords);
    mlt.setBoost(boostTerms);
    mlt.setBoostFactor(boostTermsFactor);
    //LUCENE 4 UPGRADE this mapps the 3.6 behavior (only use the first field)
    BooleanQuery bq = (BooleanQuery) mlt.like(new FastStringReader(likeText), moreLikeFields[0]);
    BooleanClause[] clauses = bq.getClauses();

    bq.setMinimumNumberShouldMatch((int) (clauses.length * percentTermsToMatch));

    bq.setBoost(getBoost());
    return bq;
}

---

From source file:org.ohdsi.usagi.UsagiSearchEngine.java

License:Apache License

public List<ScoredConcept> search(String searchTerm, boolean useMlt, Collection<Integer> filterConceptIds,
        String filterDomain, String filterConceptClass, String filterVocabulary, boolean filterInvalid) {
    List<ScoredConcept> results = new ArrayList<ScoredConcept>();
    try {/*  w  w  w . j a  v a  2s.c  o  m*/
        Query query;
        if (useMlt) {
            MoreLikeThis mlt = new MoreLikeThis(searcher.getIndexReader());
            mlt.setMinTermFreq(1);
            mlt.setMinDocFreq(1);
            mlt.setMaxDocFreq(9999);
            mlt.setMinWordLen(1);
            mlt.setMaxWordLen(9999);
            mlt.setMaxDocFreqPct(100);
            mlt.setMaxNumTokensParsed(9999);
            mlt.setMaxQueryTerms(9999);
            mlt.setStopWords(null);
            mlt.setFieldNames(new String[] { "TERM" });
            mlt.setAnalyzer(analyzer);

            query = mlt.like("TERM", new StringReader(searchTerm));
        } else {
            try {
                query = keywordsQueryParser.parse(searchTerm);
                // if (query instanceof BooleanQuery) {
                // List<BooleanClause> clauses = ((BooleanQuery) query).clauses();
                // BooleanClause lastClause = clauses.get(clauses.size() - 1);
                // lastClause.setQuery(new PrefixQuery(((TermQuery) lastClause.getQuery()).getTerm()));
                // } else if (query instanceof TermQuery) {// It's a single term
                // query = new PrefixQuery(((TermQuery) query).getTerm());
                // }

            } catch (ParseException e) {
                return results;
            }
        }

        BooleanQuery booleanQuery = new BooleanQuery();
        booleanQuery.add(query, Occur.SHOULD);
        booleanQuery.add(conceptQuery, Occur.MUST);

        if (filterConceptIds != null && filterConceptIds.size() > 0) {
            Query conceptIdQuery = conceptIdQueryParser.parse(StringUtilities.join(filterConceptIds, " OR "));
            booleanQuery.add(conceptIdQuery, Occur.MUST);
        }

        if (filterDomain != null) {
            Query domainQuery = domainQueryParser.parse("\"" + filterDomain + "\"");
            booleanQuery.add(domainQuery, Occur.MUST);
        }
        if (filterConceptClass != null) {
            Query conceptClassQuery = conceptClassQueryParser
                    .parse("\"" + filterConceptClass.toString() + "\"");
            booleanQuery.add(conceptClassQuery, Occur.MUST);
        }
        if (filterVocabulary != null) {
            Query vocabularyQuery = vocabularyQueryParser.parse("\"" + filterVocabulary.toString() + "\"");
            booleanQuery.add(vocabularyQuery, Occur.MUST);
        }
        if (filterInvalid) {
            Query invalidQuery = invalidQueryParser.parse("\"\"");
            booleanQuery.add(invalidQuery, Occur.MUST);
        }
        TopDocs topDocs = searcher.search(booleanQuery, 100);

        recomputeScores(topDocs.scoreDocs, query);
        for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
            Document document = reader.document(scoreDoc.doc);
            int conceptId = Integer.parseInt(document.get("CONCEPT_ID"));
            // If matchscore = 0 but it was the one concept that was automatically selected, still allow it:
            if (scoreDoc.score > 0 || (filterConceptIds != null && filterConceptIds.size() == 1
                    && filterConceptIds.contains(conceptId))) {
                TargetConcept targetConcept = new TargetConcept();
                targetConcept.term = document.get("TERM");
                targetConcept.conceptId = conceptId;
                targetConcept.conceptName = document.get("CONCEPT_NAME");
                targetConcept.conceptClass = document.get("CONCEPT_CLASS");
                targetConcept.vocabulary = document.get("VOCABULARY");
                targetConcept.conceptCode = document.get("CONCEPT_CODE");
                targetConcept.validStartDate = document.get("VALID_START_DATE");
                targetConcept.validEndDate = document.get("VALID_END_DATE");
                targetConcept.invalidReason = document.get("INVALID_REASON");
                for (String domain : document.get("DOMAINS").split("\n"))
                    targetConcept.domains.add(domain);
                targetConcept.additionalInformation = document.get("ADDITIONAL_INFORMATION");
                results.add(new ScoredConcept(scoreDoc.score, targetConcept));
            }
        }
        reorderTies(results);
        removeDuplicateConcepts(results);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
    }

    return results;
}

---

From source file:uk.gov.nationalarchives.discovery.taxonomy.common.service.impl.TSetBasedCategoriserServiceImpl.java

License:Mozilla Public License

/**
 * run More Like This process on a document by comparing its description to
 * the description of all items of the training set<br/>
 * currently we get a fixed number of the top results
 * /*from   ww w . j av  a 2  s. c o  m*/
 * @param document
 *            document being tested
 * @return
 * @throws IOException
 */
public List<TSetBasedCategorisationResult> runMlt(Document document) {

    Map<String, TSetBasedCategorisationResult> result = null;
    IndexSearcher searcher = null;
    try {
        trainingSetSearcherManager.maybeRefresh();
        // Boolean wasRefreshed = trainingSetSearcherManager.maybeRefresh();
        // if (wasRefreshed) {
        // logger.debug(".runMlt: training set searcher had to be refreshed");
        // }
        searcher = trainingSetSearcherManager.acquire();

        // TODO TSETBASED refresh reader/searcher: Use readermanager and
        // refresh it?
        MoreLikeThis moreLikeThis = new MoreLikeThis(this.trainingSetIndexReader);
        moreLikeThis.setMinTermFreq(minTermFreq);
        moreLikeThis.setMinDocFreq(minDocFreq);
        moreLikeThis.setAnalyzer(this.trainingSetAnalyser);
        moreLikeThis.setFieldNames(fieldsToAnalyse.split(","));
        moreLikeThis.setBoost(true);

        BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();

        for (String fieldName : fieldsToAnalyse.split(",")) {
            String value = document.get(fieldName);
            if (value != null && !"null".equals(value)) {

                switch (InformationAssetViewFields.valueOf(fieldName)) {
                case DESCRIPTION:
                    moreLikeThis.setBoostFactor(descBoostingFactor);
                    break;
                case TITLE:
                    moreLikeThis.setBoostFactor(titleBoostingFactor);
                    break;
                case CONTEXTDESCRIPTION:
                    moreLikeThis.setBoostFactor(contextDescBoostingFactor);
                    break;
                default:
                case SUBJECTS:
                case CORPBODYS:
                case PERSON_FULLNAME:
                case PLACE_NAME:
                    moreLikeThis.setBoostFactor(1);
                    break;
                }
                Query query = moreLikeThis.like(fieldName, new StringReader(value));
                queryBuilder.add(query, Occur.SHOULD);
            }
        }
        BooleanQuery fullQuery = queryBuilder.build();

        TopDocs topDocs = searcher.search(fullQuery, this.maximumSimilarElements);
        logger.debug(".runMlt: found {} total hits, processed at maximum {} hits", topDocs.totalHits,
                this.maximumSimilarElements);

        result = new LinkedHashMap<String, TSetBasedCategorisationResult>();

        int size = 0;
        if (topDocs.totalHits <= this.maximumSimilarElements) {
            size = topDocs.totalHits - 1;
        } else {
            size = this.maximumSimilarElements - 1;
        }

        for (int i = 0; i < size; i++) {
            ScoreDoc scoreDoc = topDocs.scoreDocs[i];
            Float currrentScore = scoreDoc.score;

            if (currrentScore < this.mimimumScoreForMlt) {
                break;
            }

            Document hitDoc = searcher.doc(scoreDoc.doc);
            String category = hitDoc.get(InformationAssetViewFields.TAXONOMY.toString());
            String docReference = hitDoc.get(InformationAssetViewFields.DOCREFERENCE.toString());
            logger.debug(".runMlt: found doc, category: {}, score: {}, docreference: {}", category,
                    currrentScore, docReference);

            TSetBasedCategorisationResult existingCategorisationResult = result.get(category);
            Float scoreToSet = currrentScore;
            Integer numberOfFoundDocuments = 1;
            // k nearest neighbour algorithm
            if (existingCategorisationResult != null) {
                scoreToSet += existingCategorisationResult.getScore();
                numberOfFoundDocuments += existingCategorisationResult.getNumberOfFoundDocuments();
            }
            result.put(category,
                    new TSetBasedCategorisationResult(category, scoreToSet, numberOfFoundDocuments));

        }

    } catch (IOException e) {
        throw new TaxonomyException(TaxonomyErrorType.LUCENE_IO_EXCEPTION, e);
    } finally {
        LuceneHelperTools.releaseSearcherManagerQuietly(trainingSetSearcherManager, searcher);
    }

    List<TSetBasedCategorisationResult> sortedResults = sortCategorisationResultsByScoreDescAndFilterByGlobalScore(
            new ArrayList<TSetBasedCategorisationResult>(result.values()));

    return sortedResults;
}

---

• «
• 1
• 2
• »