Example usage for org.apache.lucene.analysis TokenStream toString

Introduction

In this page you can find the example usage for org.apache.lucene.analysis TokenStream toString.

Prototype

@Override
public String toString() 

Source Link

Document

Returns a string consisting of the class's simple name, the hex representation of the identity hash code, and the current reflection of all attributes.

Usage

From source file:com.quest.agent.weibomonitor.weiboMonitorAgentImpl.java

License:Open Source License

private ModelRoot collect(long collectionFreqInMs, String groupName, String sqlQuery, int groupID) {
    Weibo agentRoot = new Weibo(groupName);
    //TODO: collect data and populate the data collected to model(topology) 

    //List<UrlList> urlList = mWrapper.getUrlList();

    Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT);

    try {/*from  ww  w .j a va 2 s. co m*/

        SQLProcess sql = new SQLProcess();

        ResultSet res = sql.executeQuery(sqlQuery);

        MyTokenFilter tkFilter = new MyTokenFilter();

        while (res.next()) {
            Reader sentence = new StringReader(res.getString("status").toString());

            String weiboID = res.getObject("weiboId").toString();

            if (groupID == 0)
                sql.execute("update status set status.read=1 where weiboId=" + weiboID + ";");
            else
                sql.execute("update status2 set status2.read=1 where weiboId=" + weiboID + ";");

            TokenStream ts = ca.tokenStream("", sentence);
            try {
                while (ts.incrementToken()) {
                    String ss[] = ts.toString().split(",");
                    ss[0] = ss[0].replace("(", "");
                    if (tkFilter.doFilter(ss[0])) {
                        if (!map[groupID].containsKey(ss[0]))
                            map[groupID].put(ss[0], new Word(1, ss[0]));
                        else
                            map[groupID].get(ss[0]).plusNum();
                    }
                }
            } catch (IOException e) {
                mLogger.debug2("error occurred while incrementToken", e);
            }
        }
    } catch (SQLException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    Word[] wordList = tfidf.doProcess(map[groupID]);

    int mapsize = map[groupID].size();
    for (int i = 0; i < Math.min(mapsize, ItemNumShow); i++) {

        collectWeibo(wordList[i].getWord(), wordList[i].getNum(), wordList[i].getTfIdf(), wordList[i].getIdf(),
                agentRoot);

    }

    return agentRoot;
}

---

From source file:lia.recent.CopyOfSearch.java

License:Apache License

  /** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {

   System.out.println("Hello");
   KoreanAnalyzer ka = new KoreanAnalyzer();
   TokenStream ts = ka.tokenStream("", new java.io.StringReader("? ? I an Hello"));
   System.out.println(ts.toString());
   try{/*from www  .  j av  a2  s. c  o  m*/
   while (ts.incrementToken()){
   org.apache.lucene.analysis.tokenattributes.TermAttribute ta = ts.getAttribute( org.apache.lucene.analysis.tokenattributes.TermAttribute.class);
   System.out.println("adf"+ta.term());
   }
   }catch (Exception e){System.out.println(e.toString());}


   }

---