Example usage for org.apache.lucene.search.highlight TokenSources getTokenStream

Introduction

In this page you can find the example usage for org.apache.lucene.search.highlight TokenSources getTokenStream.

Prototype

@Deprecated 
public static TokenStream getTokenStream(final Terms tpv) throws IOException 

Source Link

Document

Returns a token stream generated from a Terms .

Usage

From source file:com.taobao.common.tedis.support.lucene.analysis.xanalyzer.TestHighLight.java

License:Open Source License

/**
 * @param args//from  ww  w.j  a va 2s . c o m
 */
public static void main(String[] args) {

    Directory ramDir = new RAMDirectory();
    try {
        IndexWriter writer = new IndexWriter(ramDir, /*
                                                      * new
                                                      * StandardAnalyzer()/
                                                      */XFactory.getWriterAnalyzer());
        Document doc = new Document();
        Field fd = new Field(FIELD_NAME, CONTENT, Field.Store.YES, Field.Index.TOKENIZED,
                Field.TermVector.WITH_POSITIONS_OFFSETS);
        doc.add(fd);
        writer.addDocument(doc);
        writer.optimize();
        writer.close();

        IndexReader reader = IndexReader.open(ramDir);
        String queryString = QUERY;
        QueryParser parser = new QueryParser(FIELD_NAME, /*
                                                          * new
                                                          * StandardAnalyzer
                                                          * ()/
                                                          */XFactory.getWriterAnalyzer());
        Query query = parser.parse(queryString);
        System.out.println(query);
        Searcher searcher = new IndexSearcher(ramDir);
        query = query.rewrite(reader);
        System.out.println(query);
        System.out.println("Searching for: " + query.toString(FIELD_NAME));
        Hits hits = searcher.search(query);

        BoldFormatter formatter = new BoldFormatter();
        Highlighter highlighter = new Highlighter(formatter, new QueryScorer(query));
        highlighter.setTextFragmenter(new SimpleFragmenter(50));
        for (int i = 0; i < hits.length(); i++) {
            String text = hits.doc(i).get(FIELD_NAME);
            int maxNumFragmentsRequired = 5;
            String fragmentSeparator = "...";
            TermPositionVector tpv = (TermPositionVector) reader.getTermFreqVector(hits.id(i), FIELD_NAME);
            TokenStream tokenStream = TokenSources.getTokenStream(tpv);
            /*
             * TokenStream tokenStream2= (new StandardAnalyzer())
             * //XFactory.getWriterAnalyzer() .tokenStream(FIELD_NAME,new
             * StringReader(text));
             *
             * do { Token t = tokenStream2.next(); if(t==null)break;
             * System.out.println("\t" + t.startOffset() + "," +
             * t.endOffset() + "\t" + t.termText()); }while(true);
             */
            String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
                    fragmentSeparator);
            System.out.println("\n" + result);
        }
        reader.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:org.compass.core.lucene.engine.LuceneSearchEngineHighlighter.java

License:Apache License

protected TokenStream createTokenStreamFromTermPositions(Resource resource, String propertyName)
        throws SearchEngineException {
    int docId = ((LuceneResource) resource).getDocNum();
    TermFreqVector tfv;/*from  w w w . jav a  2s  .c  om*/
    try {
        tfv = indexReader.getTermFreqVector(docId, propertyName);
    } catch (IOException e) {
        throw new SearchEngineException("Failed to read term vector info", e);
    }
    if (tfv != null) {
        if (tfv instanceof TermPositionVector) {
            return TokenSources.getTokenStream((TermPositionVector) tfv);
        }
    }
    return null;
}

---