List of usage examples for org.apache.lucene.util LineFileDocs close
@Override
public synchronized void close() throws IOException
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTestCase.java
License:Apache License
public void testLineDocs() throws IOException { Random r = getRandom();//ww w . j a v a 2s. co m LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); }
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTestCase.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = getRandom(); int threadCount = TestUtil.nextInt(r, 2, 6); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override/* ww w .ja va 2 s. c om*/ public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTests.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = getRandom(); int threadCount = TestUtil.nextInt(r, 2, 10); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override/*from www. j a v a 2 s. co m*/ public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.common.compress.DeflateCompressTests.java
License:Apache License
public void testLineDocs() throws IOException { Random r = random();/* w w w . j av a2 s . c o m*/ LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); }
From source file:org.elasticsearch.common.compress.DeflateCompressTests.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = random(); int threadCount = TestUtil.nextInt(r, 2, 6); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override//from ww w .j av a2s.co m public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTest.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT), new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); LineFileDocs docs = new LineFileDocs(getRandom()); int num = atLeast(150); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);//from w ww . j a va 2s.c o m } docs.close(); final InputIterator primaryIter = new InputIterator() { int index = 0; long currentWeight = -1; @Override public Comparator<BytesRef> getComparator() { return null; } @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } @Override public BytesRef payload() { return null; } @Override public boolean hasPayloads() { return false; } }; InputIterator iter; if (usePayloads) { iter = new InputIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public Comparator<BytesRef> getComparator() { return primaryIter.getComparator(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } @Override public boolean hasPayloads() { return true; } }; } else { iter = primaryIter; } reference.build(iter); PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new Elasticsearch090PostingsFormat()); NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT)); final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer, namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements, Integer.MAX_VALUE, AbstractFieldMapper.MultiFields.empty(), null); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); refField.setAccessible(true); assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup))); for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }
From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTests.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(), null, new StandardAnalyzer(), options, 256, -1, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); LineFileDocs docs = new LineFileDocs(getRandom()); int num = scaledRandomIntBetween(150, 300); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);//ww w . j a v a 2 s.c om } docs.close(); final InputIterator primaryIter = new InputIterator() { int index = 0; long currentWeight = -1; @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } @Override public BytesRef payload() { return null; } @Override public boolean hasPayloads() { return false; } @Override public Set<BytesRef> contexts() { return null; } @Override public boolean hasContexts() { return false; } }; InputIterator iter; if (usePayloads) { iter = new InputIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } @Override public boolean hasPayloads() { return true; } @Override public Set<BytesRef> contexts() { return null; } @Override public boolean hasContexts() { return false; } }; } else { iter = primaryIter; } reference.build(iter); AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider( preserveSeparators, false, preservePositionIncrements, usePayloads); CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); final CompletionFieldMapper mapper = new CompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE, indexSettings, FieldMapper.MultiFields.empty(), null); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); if (buildAnalyzingLookup instanceof XAnalyzingSuggester) { assertEquals(reference.getMaxAnalyzedPathsForOneInput(), ((XAnalyzingSuggester) buildAnalyzingLookup).getMaxAnalyzedPathsForOneInput()); } for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(analyzer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }
From source file:org.elasticsearch.search.suggest.completion.old.CompletionPostingsFormatTest.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(), null, new StandardAnalyzer(), options, 256, -1, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); LineFileDocs docs = new LineFileDocs(getRandom()); int num = scaledRandomIntBetween(150, 300); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);/* ww w .jav a 2 s .c o m*/ } docs.close(); final InputIterator primaryIter = new InputIterator() { int index = 0; long currentWeight = -1; @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } @Override public BytesRef payload() { return null; } @Override public boolean hasPayloads() { return false; } @Override public Set<BytesRef> contexts() { return null; } @Override public boolean hasContexts() { return false; } }; InputIterator iter; if (usePayloads) { iter = new InputIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } @Override public boolean hasPayloads() { return true; } @Override public Set<BytesRef> contexts() { return null; } @Override public boolean hasContexts() { return false; } }; } else { iter = primaryIter; } reference.build(iter); AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider( preserveSeparators, false, preservePositionIncrements, usePayloads); OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); final OldCompletionFieldMapper mapper = new OldCompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE, indexSettings, FieldMapper.MultiFields.empty(), null); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); refField.setAccessible(true); assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup))); for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(analyzer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }
From source file:org.elasticsearch.search.suggest.CompletionPostingsFormatTest.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT), new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, null, false, 1); reference.setPreservePositionIncrements(preservePositionIncrements); LineFileDocs docs = new LineFileDocs(getRandom()); int num = atLeast(150); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);//from w w w . ja v a 2 s . c o m } docs.close(); final TermFreqIterator primaryIter = new TermFreqIterator() { int index = 0; long currentWeight = -1; @Override public Comparator<BytesRef> getComparator() { return null; } @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } }; TermFreqIterator iter; if (usePayloads) { iter = new TermFreqPayloadIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public Comparator<BytesRef> getComparator() { return primaryIter.getComparator(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } }; } else { iter = primaryIter; } reference.build(iter); PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new ElasticSearch090PostingsFormat()); NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT)); final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer, namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements, Integer.MAX_VALUE); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); refField.setAccessible(true); assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup))); for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }
From source file:org.elasticsearch.test.integration.search.suggest.CompletionPostingsFormatTest.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT), new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, null, false, 1); reference.setPreservePositionIncrements(preservePositionIncrements); LineFileDocs docs = new LineFileDocs(getRandom()); int num = atLeast(150); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);//from w ww . j ava 2s.c om } docs.close(); final TermFreqIterator primaryIter = new TermFreqIterator() { int index = 0; long currentWeight = -1; @Override public Comparator<BytesRef> getComparator() { return null; } @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } }; TermFreqIterator iter; if (usePayloads) { iter = new TermFreqPayloadIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public Comparator<BytesRef> getComparator() { return primaryIter.getComparator(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } }; } else { iter = primaryIter; } reference.build(iter); PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new ElasticSearch090PostingsFormat()); NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT)); final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer, namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); refField.setAccessible(true); assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup))); for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }