Example usage for org.apache.lucene.analysis.ko KoreanTokenizer KoreanTokenizer

Introduction

In this page you can find the example usage for org.apache.lucene.analysis.ko KoreanTokenizer KoreanTokenizer.

Prototype

public KoreanTokenizer() 

Source Link

Document

Creates a new KoreanTokenizer with default parameters.

Usage

From source file:org.elasticsearch.index.analysis.NoriAnalysisTests.java

License:Apache License

public void testNoriPartOfSpeech() throws IOException {
    Settings settings = Settings.builder().put("index.analysis.filter.my_filter.type", "nori_part_of_speech")
            .put("index.analysis.filter.my_filter.stoptags", "NR, SP").build();
    TestAnalysis analysis = createTestAnalysis(settings);
    TokenFilterFactory factory = analysis.tokenFilter.get("my_filter");
    Tokenizer tokenizer = new KoreanTokenizer();
    tokenizer.setReader(new StringReader(" ?"));
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "", "?" });
}

---

From source file:org.elasticsearch.index.analysis.NoriAnalysisTests.java

License:Apache License

public void testNoriReadingForm() throws IOException {
    Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
            .put("index.analysis.filter.my_filter.type", "nori_readingform").build();
    TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings,
            new AnalysisNoriPlugin());
    TokenFilterFactory factory = analysis.tokenFilter.get("my_filter");
    Tokenizer tokenizer = new KoreanTokenizer();
    tokenizer.setReader(new StringReader(""));
    TokenStream stream = factory.create(tokenizer);
    assertTokenStreamContents(stream, new String[] { "" });
}

---