Example usage for org.apache.lucene.analysis.standard UAX29URLEmailTokenizer setMaxTokenLength

Introduction

In this page you can find the example usage for org.apache.lucene.analysis.standard UAX29URLEmailTokenizer setMaxTokenLength.

Prototype

public void setMaxTokenLength(int length) 

Source Link

Document

Set the max allowed token length.

Usage

From source file:org.apache.solr.analysis.UAX29URLEmailTokenizerFactory.java

License:Apache License

public UAX29URLEmailTokenizer create(Reader input) {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, input);
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
}

---

From source file:org.elasticsearch.analysis.common.UAX29URLEmailTokenizerFactory.java

License:Apache License

@Override
public Tokenizer create() {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer();
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
}

---

From source file:org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory.java

License:Apache License

@Override
public Tokenizer create(Reader reader) {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(version, reader);
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
}

---

From source file:org.nlp.lucene.patch.UAX29URLEmailAnalyzer.java

License:Apache License

@Override
protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
    final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(matchVersion, reader);
    src.setMaxTokenLength(maxTokenLength);
    TokenStream tok = new StandardFilter(matchVersion, src);
    tok = new LowerCaseFilter(matchVersion, tok);
    tok = new StopFilter(matchVersion, tok, stopwords);
    // /*w  w w.  jav a 2s  .  c o  m*/
    return new TokenStreamComponents(src, tok) {
        @Override
        protected void setReader(final Reader reader) throws IOException {
            src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
            super.setReader(reader);
        }
    };
}

---

From source file:org.nlp.lucene.patch.UAX29URLEmailTokenizerFactory.java

License:Apache License

@Override
public Tokenizer create(AttributeFactory factory, Reader input) {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, input);
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
}

---

From source file:stackoverflow.lucene.modified.PorterUAX29URLEmailAnalyzer.java

License:Apache License

@Override
protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
    final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(matchVersion, reader);
    src.setMaxTokenLength(maxTokenLength);
    TokenStream tok = new StandardFilter(matchVersion, src);
    tok = new LowerCaseFilter(matchVersion, tok);
    tok = new StopFilter(matchVersion, tok, stopwords);
    tok = new PorterStemFilter(tok);
    return new TokenStreamComponents(src, tok) {
        @Override/*from  w w  w  .  jav  a  2  s  .com*/
        protected void setReader(final Reader reader) throws IOException {
            src.setMaxTokenLength(PorterUAX29URLEmailAnalyzer.this.maxTokenLength);
            super.setReader(reader);
        }
    };
}

---