Example usage for org.apache.lucene.search.suggest.analyzing AnalyzingSuggester PRESERVE

Introduction

In this page you can find the example usage for org.apache.lucene.search.suggest.analyzing AnalyzingSuggester PRESERVE_SEP.

Prototype

int PRESERVE_SEP

To view the source code for org.apache.lucene.search.suggest.analyzing AnalyzingSuggester PRESERVE_SEP.

Click Source Link

Document

Include this flag in the options parameter to #AnalyzingSuggester(Directory,String,Analyzer,Analyzer,int,int,int,boolean) to preserve token separators when matching.

Usage

From source file:org.apache.solr.spelling.suggest.fst.AnalyzingLookupFactory.java

License:Apache License

@Override
public Lookup create(NamedList params, SolrCore core) {
    // mandatory parameter
    Object fieldTypeName = params.get(QUERY_ANALYZER);
    if (fieldTypeName == null) {
        throw new IllegalArgumentException(
                "Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
    }//from w  ww .  j a  v a2  s.  c  o  m
    FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
    Analyzer indexAnalyzer = ft.getAnalyzer();
    Analyzer queryAnalyzer = ft.getQueryAnalyzer();

    // optional parameters

    boolean exactMatchFirst = params.get(EXACT_MATCH_FIRST) != null
            ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString())
            : true;

    boolean preserveSep = params.get(PRESERVE_SEP) != null
            ? Boolean.valueOf(params.get(PRESERVE_SEP).toString())
            : true;

    int flags = 0;
    if (exactMatchFirst) {
        flags |= AnalyzingSuggester.EXACT_FIRST;
    }
    if (preserveSep) {
        flags |= AnalyzingSuggester.PRESERVE_SEP;
    }

    int maxSurfaceFormsPerAnalyzedForm = params.get(MAX_SURFACE_FORMS) != null
            ? Integer.parseInt(params.get(MAX_SURFACE_FORMS).toString())
            : 256;

    int maxGraphExpansions = params.get(MAX_EXPANSIONS) != null
            ? Integer.parseInt(params.get(MAX_EXPANSIONS).toString())
            : -1;

    boolean preservePositionIncrements = params.get(PRESERVE_POSITION_INCREMENTS) != null
            ? Boolean.valueOf(params.get(PRESERVE_POSITION_INCREMENTS).toString())
            : false;

    return new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, flags, maxSurfaceFormsPerAnalyzedForm,
            maxGraphExpansions, preservePositionIncrements);
}

From source file:org.apache.solr.spelling.suggest.SmartAnalyzingLookupFactory.java

License:Apache License

@Override
public Lookup create(NamedList params, SolrCore core) {
    System.out.println("====> SmartAnalyzingLookupFactory.create() : Lookup called");

    // mandatory parameter
    Object fieldTypeName = params.get(QUERY_ANALYZER);
    if (fieldTypeName == null) {
        throw new IllegalArgumentException(
                "Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
    }/*  www .jav a 2  s  . co m*/
    FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
    if (ft == null) {
        throw new IllegalArgumentException(
                "Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
    }

    Analyzer indexAnalyzer = ft.getIndexAnalyzer();
    Analyzer queryAnalyzer = ft.getQueryAnalyzer();

    // optional parameters

    boolean exactMatchFirst = params.get(EXACT_MATCH_FIRST) != null
            ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString())
            : true;

    boolean preserveSep = params.get(PRESERVE_SEP) != null
            ? Boolean.valueOf(params.get(PRESERVE_SEP).toString())
            : true;

    int flags = 0;
    if (exactMatchFirst) {
        flags |= AnalyzingSuggester.EXACT_FIRST;
    }
    if (preserveSep) {
        flags |= AnalyzingSuggester.PRESERVE_SEP;
    }

    int maxSurfaceFormsPerAnalyzedForm = params.get(MAX_SURFACE_FORMS) != null
            ? Integer.parseInt(params.get(MAX_SURFACE_FORMS).toString())
            : 256;

    int maxGraphExpansions = params.get(MAX_EXPANSIONS) != null
            ? Integer.parseInt(params.get(MAX_EXPANSIONS).toString())
            : -1;

    boolean preservePositionIncrements = params.get(PRESERVE_POSITION_INCREMENTS) != null
            ? Boolean.valueOf(params.get(PRESERVE_POSITION_INCREMENTS).toString())
            : false;

    return new SmartAnalyzingSuggester(indexAnalyzer, queryAnalyzer, flags, maxSurfaceFormsPerAnalyzedForm,
            maxGraphExpansions, preservePositionIncrements);
}

From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTest.java

License:Apache License

@Test
public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException,
        IllegalArgumentException, IllegalAccessException {
    final boolean preserveSeparators = getRandom().nextBoolean();
    final boolean preservePositionIncrements = getRandom().nextBoolean();
    final boolean usePayloads = getRandom().nextBoolean();
    final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0;

    XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT),
            new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, preservePositionIncrements, null,
            false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP,
            XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER);
    LineFileDocs docs = new LineFileDocs(getRandom());
    int num = atLeast(150);
    final String[] titles = new String[num];
    final long[] weights = new long[num];
    for (int i = 0; i < titles.length; i++) {
        Document nextDoc = docs.nextDoc();
        IndexableField field = nextDoc.getField("title");
        titles[i] = field.stringValue();
        weights[i] = between(0, 100);//  w ww .  j a v a 2 s . com

    }
    docs.close();
    final InputIterator primaryIter = new InputIterator() {
        int index = 0;
        long currentWeight = -1;

        @Override
        public Comparator<BytesRef> getComparator() {
            return null;
        }

        @Override
        public BytesRef next() throws IOException {
            if (index < titles.length) {
                currentWeight = weights[index];
                return new BytesRef(titles[index++]);
            }
            return null;
        }

        @Override
        public long weight() {
            return currentWeight;
        }

        @Override
        public BytesRef payload() {
            return null;
        }

        @Override
        public boolean hasPayloads() {
            return false;
        }

    };
    InputIterator iter;
    if (usePayloads) {
        iter = new InputIterator() {
            @Override
            public long weight() {
                return primaryIter.weight();
            }

            @Override
            public Comparator<BytesRef> getComparator() {
                return primaryIter.getComparator();
            }

            @Override
            public BytesRef next() throws IOException {
                return primaryIter.next();
            }

            @Override
            public BytesRef payload() {
                return new BytesRef(Long.toString(weight()));
            }

            @Override
            public boolean hasPayloads() {
                return true;
            }
        };
    } else {
        iter = primaryIter;
    }
    reference.build(iter);
    PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new Elasticsearch090PostingsFormat());

    NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT));
    final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer,
            namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements,
            Integer.MAX_VALUE, AbstractFieldMapper.MultiFields.empty(), null);
    Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights);
    Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    field.setAccessible(true);
    Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    refField.setAccessible(true);
    assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup)));

    for (int i = 0; i < titles.length; i++) {
        int res = between(1, 10);
        final StringBuilder builder = new StringBuilder();
        SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                if (builder.length() == 0) {
                    builder.append(this.charTermAttr.toString());
                }
            }
        });
        String firstTerm = builder.toString();
        String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length()));
        List<LookupResult> refLookup = reference.lookup(prefix, false, res);
        List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res);
        assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size()));
        for (int j = 0; j < refLookup.size(); j++) {
            assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key));
            assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - "
                    + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value,
                    lookup.get(j).value, equalTo(refLookup.get(j).value));
            assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload));
            if (usePayloads) {
                assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value)));
            }
        }
    }
}

From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTests.java

License:Apache License

@Test
public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException,
        IllegalArgumentException, IllegalAccessException {
    final boolean preserveSeparators = getRandom().nextBoolean();
    final boolean preservePositionIncrements = getRandom().nextBoolean();
    final boolean usePayloads = getRandom().nextBoolean();
    final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0;

    XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(), null,
            new StandardAnalyzer(), options, 256, -1, preservePositionIncrements, null, false, 1,
            XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE,
            XAnalyzingSuggester.HOLE_CHARACTER);
    LineFileDocs docs = new LineFileDocs(getRandom());
    int num = scaledRandomIntBetween(150, 300);
    final String[] titles = new String[num];
    final long[] weights = new long[num];
    for (int i = 0; i < titles.length; i++) {
        Document nextDoc = docs.nextDoc();
        IndexableField field = nextDoc.getField("title");
        titles[i] = field.stringValue();
        weights[i] = between(0, 100);/*from   w ww . ja  v a  2s  . c  o m*/

    }
    docs.close();
    final InputIterator primaryIter = new InputIterator() {
        int index = 0;
        long currentWeight = -1;

        @Override
        public BytesRef next() throws IOException {
            if (index < titles.length) {
                currentWeight = weights[index];
                return new BytesRef(titles[index++]);
            }
            return null;
        }

        @Override
        public long weight() {
            return currentWeight;
        }

        @Override
        public BytesRef payload() {
            return null;
        }

        @Override
        public boolean hasPayloads() {
            return false;
        }

        @Override
        public Set<BytesRef> contexts() {
            return null;
        }

        @Override
        public boolean hasContexts() {
            return false;
        }

    };
    InputIterator iter;
    if (usePayloads) {
        iter = new InputIterator() {
            @Override
            public long weight() {
                return primaryIter.weight();
            }

            @Override
            public BytesRef next() throws IOException {
                return primaryIter.next();
            }

            @Override
            public BytesRef payload() {
                return new BytesRef(Long.toString(weight()));
            }

            @Override
            public boolean hasPayloads() {
                return true;
            }

            @Override
            public Set<BytesRef> contexts() {
                return null;
            }

            @Override
            public boolean hasContexts() {
                return false;
            }
        };
    } else {
        iter = primaryIter;
    }
    reference.build(iter);

    AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(
            preserveSeparators, false, preservePositionIncrements, usePayloads);
    CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone();
    fieldType.setProvider(currentProvider);
    final CompletionFieldMapper mapper = new CompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE,
            indexSettings, FieldMapper.MultiFields.empty(), null);
    Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights);
    if (buildAnalyzingLookup instanceof XAnalyzingSuggester) {
        assertEquals(reference.getMaxAnalyzedPathsForOneInput(),
                ((XAnalyzingSuggester) buildAnalyzingLookup).getMaxAnalyzedPathsForOneInput());
    }

    for (int i = 0; i < titles.length; i++) {
        int res = between(1, 10);
        final StringBuilder builder = new StringBuilder();
        SuggestUtils.analyze(analyzer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                if (builder.length() == 0) {
                    builder.append(this.charTermAttr.toString());
                }
            }
        });
        String firstTerm = builder.toString();
        String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length()));
        List<LookupResult> refLookup = reference.lookup(prefix, false, res);
        List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res);
        assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size()));
        for (int j = 0; j < refLookup.size(); j++) {
            assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key));
            assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - "
                    + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value,
                    lookup.get(j).value, equalTo(refLookup.get(j).value));
            assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload));
            if (usePayloads) {
                assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value)));
            }
        }
    }
}

From source file:org.elasticsearch.search.suggest.completion.old.CompletionPostingsFormatTest.java

License:Apache License

@Test
public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException,
        IllegalArgumentException, IllegalAccessException {
    final boolean preserveSeparators = getRandom().nextBoolean();
    final boolean preservePositionIncrements = getRandom().nextBoolean();
    final boolean usePayloads = getRandom().nextBoolean();
    final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0;

    XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(), null,
            new StandardAnalyzer(), options, 256, -1, preservePositionIncrements, null, false, 1,
            XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE,
            XAnalyzingSuggester.HOLE_CHARACTER);
    LineFileDocs docs = new LineFileDocs(getRandom());
    int num = scaledRandomIntBetween(150, 300);
    final String[] titles = new String[num];
    final long[] weights = new long[num];
    for (int i = 0; i < titles.length; i++) {
        Document nextDoc = docs.nextDoc();
        IndexableField field = nextDoc.getField("title");
        titles[i] = field.stringValue();
        weights[i] = between(0, 100);/*from  w  ww .  ja  va  2  s . c  o m*/

    }
    docs.close();
    final InputIterator primaryIter = new InputIterator() {
        int index = 0;
        long currentWeight = -1;

        @Override
        public BytesRef next() throws IOException {
            if (index < titles.length) {
                currentWeight = weights[index];
                return new BytesRef(titles[index++]);
            }
            return null;
        }

        @Override
        public long weight() {
            return currentWeight;
        }

        @Override
        public BytesRef payload() {
            return null;
        }

        @Override
        public boolean hasPayloads() {
            return false;
        }

        @Override
        public Set<BytesRef> contexts() {
            return null;
        }

        @Override
        public boolean hasContexts() {
            return false;
        }

    };
    InputIterator iter;
    if (usePayloads) {
        iter = new InputIterator() {
            @Override
            public long weight() {
                return primaryIter.weight();
            }

            @Override
            public BytesRef next() throws IOException {
                return primaryIter.next();
            }

            @Override
            public BytesRef payload() {
                return new BytesRef(Long.toString(weight()));
            }

            @Override
            public boolean hasPayloads() {
                return true;
            }

            @Override
            public Set<BytesRef> contexts() {
                return null;
            }

            @Override
            public boolean hasContexts() {
                return false;
            }
        };
    } else {
        iter = primaryIter;
    }
    reference.build(iter);

    AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(
            preserveSeparators, false, preservePositionIncrements, usePayloads);
    OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone();
    fieldType.setProvider(currentProvider);
    final OldCompletionFieldMapper mapper = new OldCompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE,
            indexSettings, FieldMapper.MultiFields.empty(), null);
    Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights);
    Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    field.setAccessible(true);
    Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    refField.setAccessible(true);
    assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup)));

    for (int i = 0; i < titles.length; i++) {
        int res = between(1, 10);
        final StringBuilder builder = new StringBuilder();
        SuggestUtils.analyze(analyzer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                if (builder.length() == 0) {
                    builder.append(this.charTermAttr.toString());
                }
            }
        });
        String firstTerm = builder.toString();
        String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length()));
        List<LookupResult> refLookup = reference.lookup(prefix, false, res);
        List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res);
        assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size()));
        for (int j = 0; j < refLookup.size(); j++) {
            assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key));
            assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - "
                    + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value,
                    lookup.get(j).value, equalTo(refLookup.get(j).value));
            assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload));
            if (usePayloads) {
                assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value)));
            }
        }
    }
}

From source file:org.elasticsearch.search.suggest.CompletionPostingsFormatTest.java

License:Apache License

@Test
public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException,
        IllegalArgumentException, IllegalAccessException {
    final boolean preserveSeparators = getRandom().nextBoolean();
    final boolean preservePositionIncrements = getRandom().nextBoolean();
    final boolean usePayloads = getRandom().nextBoolean();
    final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0;

    XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT),
            new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, null, false, 1);
    reference.setPreservePositionIncrements(preservePositionIncrements);
    LineFileDocs docs = new LineFileDocs(getRandom());
    int num = atLeast(150);
    final String[] titles = new String[num];
    final long[] weights = new long[num];
    for (int i = 0; i < titles.length; i++) {
        Document nextDoc = docs.nextDoc();
        IndexableField field = nextDoc.getField("title");
        titles[i] = field.stringValue();
        weights[i] = between(0, 100);//w  w w . j  ava  2 s.co m

    }
    docs.close();
    final TermFreqIterator primaryIter = new TermFreqIterator() {
        int index = 0;
        long currentWeight = -1;

        @Override
        public Comparator<BytesRef> getComparator() {
            return null;
        }

        @Override
        public BytesRef next() throws IOException {
            if (index < titles.length) {
                currentWeight = weights[index];
                return new BytesRef(titles[index++]);
            }
            return null;
        }

        @Override
        public long weight() {
            return currentWeight;
        }

    };
    TermFreqIterator iter;
    if (usePayloads) {
        iter = new TermFreqPayloadIterator() {
            @Override
            public long weight() {
                return primaryIter.weight();
            }

            @Override
            public Comparator<BytesRef> getComparator() {
                return primaryIter.getComparator();
            }

            @Override
            public BytesRef next() throws IOException {
                return primaryIter.next();
            }

            @Override
            public BytesRef payload() {
                return new BytesRef(Long.toString(weight()));
            }
        };
    } else {
        iter = primaryIter;
    }
    reference.build(iter);
    PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new ElasticSearch090PostingsFormat());

    NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT));
    final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer,
            namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements,
            Integer.MAX_VALUE);
    Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights);
    Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    field.setAccessible(true);
    Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    refField.setAccessible(true);
    assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup)));

    for (int i = 0; i < titles.length; i++) {
        int res = between(1, 10);
        final StringBuilder builder = new StringBuilder();
        SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                if (builder.length() == 0) {
                    builder.append(this.charTermAttr.toString());
                }
            }
        });
        String firstTerm = builder.toString();
        String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length()));
        List<LookupResult> refLookup = reference.lookup(prefix, false, res);
        List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res);
        assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size()));
        for (int j = 0; j < refLookup.size(); j++) {
            assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key));
            assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - "
                    + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value,
                    lookup.get(j).value, equalTo(refLookup.get(j).value));
            assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload));
            if (usePayloads) {
                assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value)));
            }
        }
    }
}

From source file:org.elasticsearch.test.integration.search.suggest.CompletionPostingsFormatTest.java

License:Apache License

@Test
public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException,
        IllegalArgumentException, IllegalAccessException {
    final boolean preserveSeparators = getRandom().nextBoolean();
    final boolean preservePositionIncrements = getRandom().nextBoolean();
    final boolean usePayloads = getRandom().nextBoolean();
    final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0;

    XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT),
            new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, null, false, 1);
    reference.setPreservePositionIncrements(preservePositionIncrements);
    LineFileDocs docs = new LineFileDocs(getRandom());
    int num = atLeast(150);
    final String[] titles = new String[num];
    final long[] weights = new long[num];
    for (int i = 0; i < titles.length; i++) {
        Document nextDoc = docs.nextDoc();
        IndexableField field = nextDoc.getField("title");
        titles[i] = field.stringValue();
        weights[i] = between(0, 100);// w  ww  . j  a  v  a  2  s.  co  m

    }
    docs.close();
    final TermFreqIterator primaryIter = new TermFreqIterator() {
        int index = 0;
        long currentWeight = -1;

        @Override
        public Comparator<BytesRef> getComparator() {
            return null;
        }

        @Override
        public BytesRef next() throws IOException {
            if (index < titles.length) {
                currentWeight = weights[index];
                return new BytesRef(titles[index++]);
            }
            return null;
        }

        @Override
        public long weight() {
            return currentWeight;
        }

    };
    TermFreqIterator iter;
    if (usePayloads) {
        iter = new TermFreqPayloadIterator() {
            @Override
            public long weight() {
                return primaryIter.weight();
            }

            @Override
            public Comparator<BytesRef> getComparator() {
                return primaryIter.getComparator();
            }

            @Override
            public BytesRef next() throws IOException {
                return primaryIter.next();
            }

            @Override
            public BytesRef payload() {
                return new BytesRef(Long.toString(weight()));
            }
        };
    } else {
        iter = primaryIter;
    }
    reference.build(iter);
    PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new ElasticSearch090PostingsFormat());

    NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT));
    final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer,
            namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements);
    Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights);
    Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    field.setAccessible(true);
    Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput");
    refField.setAccessible(true);
    assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup)));

    for (int i = 0; i < titles.length; i++) {
        int res = between(1, 10);
        final StringBuilder builder = new StringBuilder();
        SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                if (builder.length() == 0) {
                    builder.append(this.charTermAttr.toString());
                }
            }
        });
        String firstTerm = builder.toString();
        String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length()));
        List<LookupResult> refLookup = reference.lookup(prefix, false, res);
        List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res);
        assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size()));
        for (int j = 0; j < refLookup.size(); j++) {
            assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key));
            assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - "
                    + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value,
                    lookup.get(j).value, equalTo(refLookup.get(j).value));
            assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload));
            if (usePayloads) {
                assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value)));
            }
        }
    }
}

Example usage for org.apache.lucene.search.suggest.analyzing AnalyzingSuggester PRESERVE_SEP

Introduction

Prototype

Document

Usage