List of usage examples for org.apache.lucene.util LineFileDocs nextDoc
public Document nextDoc() throws IOException
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTestCase.java
License:Apache License
public void testLineDocs() throws IOException { Random r = getRandom();//from ww w . j av a 2s. c o m LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); }
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTestCase.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = getRandom(); int threadCount = TestUtil.nextInt(r, 2, 6); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override//from www .j a v a2 s . c om public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTestCase.java
License:Apache License
private void addString(LineFileDocs lineFileDocs, ByteArrayOutputStream bos) throws IOException { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); }
From source file:org.elasticsearch.common.compress.AbstractCompressedStreamTests.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = getRandom(); int threadCount = TestUtil.nextInt(r, 2, 10); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override//from w ww . ja v a 2 s . c o m public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.common.compress.DeflateCompressTests.java
License:Apache License
public void testLineDocs() throws IOException { Random r = random();/*from w ww.j a va 2 s. c o m*/ LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); }
From source file:org.elasticsearch.common.compress.DeflateCompressTests.java
License:Apache License
public void testLineDocsThreads() throws Exception { final Random r = random(); int threadCount = TestUtil.nextInt(r, 2, 6); Thread[] threads = new Thread[threadCount]; final CountDownLatch startingGun = new CountDownLatch(1); for (int tid = 0; tid < threadCount; tid++) { final long seed = r.nextLong(); threads[tid] = new Thread() { @Override// w w w . ja v a2s. c o m public void run() { try { Random r = new Random(seed); startingGun.await(); LineFileDocs lineFileDocs = new LineFileDocs(r); for (int i = 0; i < 10; i++) { int numDocs = TestUtil.nextInt(r, 1, 200); ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int j = 0; j < numDocs; j++) { String s = lineFileDocs.nextDoc().get("body"); bos.write(s.getBytes(StandardCharsets.UTF_8)); } doTest(bos.toByteArray()); } lineFileDocs.close(); } catch (Exception e) { throw new RuntimeException(e); } } }; threads[tid].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } }
From source file:org.elasticsearch.index.translog.TranslogTests.java
License:Apache License
public void testTranslogOpsCountIsCorrect() throws IOException { List<Translog.Location> locations = new ArrayList<>(); int numOps = randomIntBetween(100, 200); LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { locations.add(translog.add(new Translog.Index("test", "" + opsAdded, lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8"))))); Translog.Snapshot snapshot = this.translog.newSnapshot(); assertEquals(opsAdded + 1, snapshot.totalOperations()); for (int i = 0; i < opsAdded; i++) { assertEquals("expected operation" + i + " to be in the current translog but wasn't", translog.currentFileGeneration(), locations.get(i).generation); Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); }/*www. ja v a 2s. c om*/ } }
From source file:org.elasticsearch.index.translog.TranslogTests.java
License:Apache License
public void testTragicEventCanBeAnyException() throws IOException { Path tempDir = createTempDir(); final FailSwitch fail = new FailSwitch(); TranslogConfig config = getTranslogConfig(tempDir); Translog translog = getFailableTranslog(fail, config, false, true, null); LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly translog.add(new Translog.Index("test", "1", lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); fail.failAlways();/*from w w w .j a va 2s . c om*/ try { Translog.Location location = translog.add(new Translog.Index("test", "2", lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))); if (randomBoolean()) { translog.ensureSynced(location); } else { translog.sync(); } //TODO once we have a mock FS that can simulate we can also fail on plain sync fail("WTF"); } catch (UnknownException ex) { // w00t } catch (TranslogException ex) { assertTrue(ex.getCause() instanceof UnknownException); } assertFalse(translog.isOpen()); assertTrue(translog.getTragicException() instanceof UnknownException); }
From source file:org.elasticsearch.index.translog.TranslogTests.java
License:Apache License
/** * This test adds operations to the translog which might randomly throw an IOException. The only thing this test verifies is * that we can, after we hit an exception, open and recover the translog successfully and retrieve all successfully synced operations * from the transaction log./* w w w . ja v a2s .c om*/ */ public void testWithRandomException() throws IOException { final int runs = randomIntBetween(5, 10); for (int run = 0; run < runs; run++) { Path tempDir = createTempDir(); final FailSwitch fail = new FailSwitch(); fail.failRandomly(); TranslogConfig config = getTranslogConfig(tempDir); final int numOps = randomIntBetween(100, 200); List<String> syncedDocs = new ArrayList<>(); List<String> unsynced = new ArrayList<>(); if (randomBoolean()) { fail.onceFailedFailAlways(); } Translog.TranslogGeneration generation = null; try { final Translog failableTLog = getFailableTranslog(fail, config, randomBoolean(), false, generation); try { LineFileDocs lineFileDocs = new LineFileDocs(random()); //writes pretty big docs so we cross buffer boarders regularly for (int opsAdded = 0; opsAdded < numOps; opsAdded++) { String doc = lineFileDocs.nextDoc().toString(); failableTLog.add( new Translog.Index("test", "" + opsAdded, doc.getBytes(Charset.forName("UTF-8")))); unsynced.add(doc); if (randomBoolean()) { failableTLog.sync(); syncedDocs.addAll(unsynced); unsynced.clear(); } if (randomFloat() < 0.1) { failableTLog.sync(); // we have to sync here first otherwise we don't know if the sync succeeded if the commit fails syncedDocs.addAll(unsynced); unsynced.clear(); if (randomBoolean()) { failableTLog.prepareCommit(); } failableTLog.commit(); syncedDocs.clear(); } } // we survived all the randomness!!! // lets close the translog and if it succeeds we are all synced again. If we don't do this we will close // it in the finally block but miss to copy over unsynced docs to syncedDocs and fail the assertion down the road... failableTLog.close(); syncedDocs.addAll(unsynced); unsynced.clear(); } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) { // fair enough } catch (IOException ex) { assertEquals(ex.getMessage(), "__FAKE__ no space left on device"); } finally { Checkpoint checkpoint = failableTLog.readCheckpoint(); if (checkpoint.numOps == unsynced.size() + syncedDocs.size()) { syncedDocs.addAll(unsynced); // failed in fsync but got fully written unsynced.clear(); } generation = failableTLog.getGeneration(); IOUtils.closeWhileHandlingException(failableTLog); } } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) { // failed - that's ok, we didn't even create it } catch (IOException ex) { assertEquals(ex.getMessage(), "__FAKE__ no space left on device"); } // now randomly open this failing tlog again just to make sure we can also recover from failing during recovery if (randomBoolean()) { try { IOUtils.close(getFailableTranslog(fail, config, randomBoolean(), false, generation)); } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) { // failed - that's ok, we didn't even create it } catch (IOException ex) { assertEquals(ex.getMessage(), "__FAKE__ no space left on device"); } } fail.failNever(); // we don't wanna fail here but we might since we write a new checkpoint and create a new tlog file try (Translog translog = new Translog(config, generation)) { Translog.Snapshot snapshot = translog.newSnapshot(); assertEquals(syncedDocs.size(), snapshot.totalOperations()); for (int i = 0; i < syncedDocs.size(); i++) { Translog.Operation next = snapshot.next(); assertEquals(syncedDocs.get(i), next.getSource().source.utf8ToString()); assertNotNull("operation " + i + " must be non-null", next); } } } }
From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTest.java
License:Apache License
@Test public void testDuellCompletions() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { final boolean preserveSeparators = getRandom().nextBoolean(); final boolean preservePositionIncrements = getRandom().nextBoolean(); final boolean usePayloads = getRandom().nextBoolean(); final int options = preserveSeparators ? AnalyzingSuggester.PRESERVE_SEP : 0; XAnalyzingSuggester reference = new XAnalyzingSuggester(new StandardAnalyzer(TEST_VERSION_CURRENT), new StandardAnalyzer(TEST_VERSION_CURRENT), options, 256, -1, preservePositionIncrements, null, false, 1, XAnalyzingSuggester.SEP_LABEL, XAnalyzingSuggester.PAYLOAD_SEP, XAnalyzingSuggester.END_BYTE, XAnalyzingSuggester.HOLE_CHARACTER); LineFileDocs docs = new LineFileDocs(getRandom()); int num = atLeast(150); final String[] titles = new String[num]; final long[] weights = new long[num]; for (int i = 0; i < titles.length; i++) { Document nextDoc = docs.nextDoc(); IndexableField field = nextDoc.getField("title"); titles[i] = field.stringValue(); weights[i] = between(0, 100);// w ww . j a v a 2 s . c o m } docs.close(); final InputIterator primaryIter = new InputIterator() { int index = 0; long currentWeight = -1; @Override public Comparator<BytesRef> getComparator() { return null; } @Override public BytesRef next() throws IOException { if (index < titles.length) { currentWeight = weights[index]; return new BytesRef(titles[index++]); } return null; } @Override public long weight() { return currentWeight; } @Override public BytesRef payload() { return null; } @Override public boolean hasPayloads() { return false; } }; InputIterator iter; if (usePayloads) { iter = new InputIterator() { @Override public long weight() { return primaryIter.weight(); } @Override public Comparator<BytesRef> getComparator() { return primaryIter.getComparator(); } @Override public BytesRef next() throws IOException { return primaryIter.next(); } @Override public BytesRef payload() { return new BytesRef(Long.toString(weight())); } @Override public boolean hasPayloads() { return true; } }; } else { iter = primaryIter; } reference.build(iter); PostingsFormatProvider provider = new PreBuiltPostingsFormatProvider(new Elasticsearch090PostingsFormat()); NamedAnalyzer namedAnalzyer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT)); final CompletionFieldMapper mapper = new CompletionFieldMapper(new Names("foo"), namedAnalzyer, namedAnalzyer, provider, null, usePayloads, preserveSeparators, preservePositionIncrements, Integer.MAX_VALUE, AbstractFieldMapper.MultiFields.empty(), null); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); Field refField = reference.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); refField.setAccessible(true); assertThat(refField.get(reference), equalTo(field.get(buildAnalyzingLookup))); for (int i = 0; i < titles.length; i++) { int res = between(1, 10); final StringBuilder builder = new StringBuilder(); SuggestUtils.analyze(namedAnalzyer.tokenStream("foo", titles[i]), new SuggestUtils.TokenConsumer() { @Override public void nextToken() throws IOException { if (builder.length() == 0) { builder.append(this.charTermAttr.toString()); } } }); String firstTerm = builder.toString(); String prefix = firstTerm.isEmpty() ? "" : firstTerm.substring(0, between(1, firstTerm.length())); List<LookupResult> refLookup = reference.lookup(prefix, false, res); List<LookupResult> lookup = buildAnalyzingLookup.lookup(prefix, false, res); assertThat(refLookup.toString(), lookup.size(), equalTo(refLookup.size())); for (int j = 0; j < refLookup.size(); j++) { assertThat(lookup.get(j).key, equalTo(refLookup.get(j).key)); assertThat("prefix: " + prefix + " " + j + " -- missmatch cost: " + lookup.get(j).key + " - " + lookup.get(j).value + " | " + refLookup.get(j).key + " - " + refLookup.get(j).value, lookup.get(j).value, equalTo(refLookup.get(j).value)); assertThat(lookup.get(j).payload, equalTo(refLookup.get(j).payload)); if (usePayloads) { assertThat(lookup.get(j).payload.utf8ToString(), equalTo(Long.toString(lookup.get(j).value))); } } } }