List of usage examples for org.apache.lucene.index RandomIndexWriter commit
public long commit() throws IOException
From source file:com.meizu.nlp.classification.utils.DocToDoubleVectorUtilsTest.java
License:Apache License
@Override @Before//w ww .j a va 2 s . c o m public void setUp() throws Exception { super.setUp(); dir = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir); FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(true); ft.setStoreTermVectorOffsets(true); ft.setStoreTermVectorPositions(true); Document doc; for (int i = 0; i < 10; i++) { doc = new Document(); doc.add(new Field("id", Integer.toString(i), ft)); doc.add(new Field("text", random().nextInt(10) + " " + random().nextInt(10) + " " + random().nextInt(10), ft)); indexWriter.addDocument(doc); } indexWriter.commit(); index = indexWriter.getReader(); indexWriter.close(); }
From source file:com.sindicetech.siren.util.SirenTestCase.java
License:Open Source License
protected static void addDocument(final RandomIndexWriter writer, final String data) throws IOException { final Document doc = new Document(); doc.add(new Field(DEFAULT_TEST_FIELD, data, newStoredFieldType())); writer.addDocument(doc);//from w ww .j av a 2 s . c o m writer.commit(); }
From source file:com.sindicetech.siren.util.SirenTestCase.java
License:Open Source License
protected void addDocumentNoNorms(final RandomIndexWriter writer, final String data) throws IOException { final Document doc = new Document(); doc.add(new Field(DEFAULT_TEST_FIELD, data, this.newStoredNoNormFieldType())); writer.addDocument(doc);/* www. jav a 2 s. co m*/ writer.commit(); }
From source file:com.sindicetech.siren.util.SirenTestCase.java
License:Open Source License
/** * Atomically adds a block of documents with sequentially * assigned document IDs./*from w w w . j a v a2 s . c o m*/ * <br> * See also {@link IndexWriter#addDocuments(Iterable)} */ protected static void addDocuments(final RandomIndexWriter writer, final String[] data) throws IOException { final ArrayList<Document> docs = new ArrayList<Document>(); for (final String entry : data) { final Document doc = new Document(); doc.add(new Field(DEFAULT_TEST_FIELD, entry, newStoredFieldType())); docs.add(doc); } writer.addDocuments(docs); writer.commit(); }
From source file:com.sindicetech.siren.util.SirenTestCase.java
License:Open Source License
protected static void addDocuments(final RandomIndexWriter writer, final MockSirenDocument... sdocs) throws IOException { final ArrayList<Document> docs = new ArrayList<Document>(sdocs.length); for (final MockSirenDocument sdoc : sdocs) { final Document doc = new Document(); doc.add(new Field(DEFAULT_TEST_FIELD, new MockSirenReader(sdoc), newFieldType())); docs.add(doc);/*ww w .j a v a2s .co m*/ } writer.addDocuments(docs); writer.commit(); }
From source file:com.sindicetech.siren.util.SirenTestCase.java
License:Open Source License
protected void deleteAll(final RandomIndexWriter writer) throws IOException { writer.deleteAll();/* w w w . j av a2 s.c o m*/ writer.commit(); }
From source file:org.apache.solr.index.TestSlowCompositeReaderWrapper.java
License:Apache License
public void testCoreListenerOnSlowCompositeReaderWrapper() throws IOException { RandomIndexWriter w = new RandomIndexWriter(random(), newDirectory()); final int numDocs = TestUtil.nextInt(random(), 1, 5); for (int i = 0; i < numDocs; ++i) { w.addDocument(new Document()); if (random().nextBoolean()) { w.commit(); }/*from w w w. j av a2 s .com*/ } w.commit(); w.close(); final IndexReader reader = DirectoryReader.open(w.w.getDirectory()); final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader); final int numListeners = TestUtil.nextInt(random(), 1, 10); final List<LeafReader.CoreClosedListener> listeners = new ArrayList<>(); AtomicInteger counter = new AtomicInteger(numListeners); for (int i = 0; i < numListeners; ++i) { CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheKey()); listeners.add(listener); leafReader.addCoreClosedListener(listener); } for (int i = 0; i < 100; ++i) { leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size()))); } final int removed = random().nextInt(numListeners); Collections.shuffle(listeners, random()); for (int i = 0; i < removed; ++i) { leafReader.removeCoreClosedListener(listeners.get(i)); } assertEquals(numListeners, counter.get()); // make sure listeners are registered on the wrapped reader and that closing any of them has the same effect if (random().nextBoolean()) { reader.close(); } else { leafReader.close(); } assertEquals(removed, counter.get()); w.w.getDirectory().close(); }
From source file:org.apache.solr.search.TestQueryWrapperFilter.java
License:Apache License
public void testScore() throws IOException { Directory dir = newDirectory();/*from ww w . j av a 2 s.c o m*/ RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new StringField("foo", "bar", Store.NO)); writer.addDocument(doc); writer.commit(); final IndexReader reader = writer.getReader(); writer.close(); final IndexSearcher searcher = new IndexSearcher(reader); final Query query = new QueryWrapperFilter(new TermQuery(new Term("foo", "bar"))); final TopDocs topDocs = searcher.search(query, 1); assertEquals(1, topDocs.totalHits); assertEquals(0f, topDocs.scoreDocs[0].score, 0f); reader.close(); dir.close(); }
From source file:org.apache.solr.search.TestQueryWrapperFilter.java
License:Apache License
public void testQueryWrapperFilterPropagatesApproximations() throws IOException { Directory dir = newDirectory();// ww w . ja v a2 s . co m RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new StringField("foo", "bar", Store.NO)); writer.addDocument(doc); writer.commit(); final IndexReader reader = writer.getReader(); writer.close(); final IndexSearcher searcher = new IndexSearcher(reader); searcher.setQueryCache(null); // to still have approximations final Query query = new QueryWrapperFilter( new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random())); final Weight weight = searcher.createNormalizedWeight(query, random().nextBoolean()); final Scorer scorer = weight.scorer(reader.leaves().get(0)); assertNotNull(scorer.twoPhaseIterator()); reader.close(); dir.close(); }
From source file:org.apache.solr.search.TestStressLucene.java
License:Apache License
@Test public void testStressLuceneNRT() throws Exception { final int commitPercent = 5 + random().nextInt(20); final int softCommitPercent = 30 + random().nextInt(75); // what percent of the commits are soft final int deletePercent = 4 + random().nextInt(25); final int deleteByQueryPercent = 1 + random().nextInt(5); final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)); int nWriteThreads = 5 + random().nextInt(25); final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers final AtomicLong operations = new AtomicLong(100000); // number of query operations to perform in total int nReadThreads = 5 + random().nextInt(25); final boolean tombstones = random().nextBoolean(); final boolean syncCommits = random().nextBoolean(); verbose("commitPercent=", commitPercent); verbose("softCommitPercent=", softCommitPercent); verbose("deletePercent=", deletePercent); verbose("deleteByQueryPercent=", deleteByQueryPercent); verbose("ndocs=", ndocs); verbose("nWriteThreads=", nWriteThreads); verbose("nReadThreads=", nReadThreads); verbose("maxConcurrentCommits=", maxConcurrentCommits); verbose("operations=", operations); verbose("tombstones=", tombstones); verbose("syncCommits=", syncCommits); initModel(ndocs);//from w ww . ja v a 2 s . com final AtomicInteger numCommitting = new AtomicInteger(); List<Thread> threads = new ArrayList<Thread>(); final FieldType idFt = new FieldType(); idFt.setIndexed(true); idFt.setStored(true); idFt.setOmitNorms(true); idFt.setTokenized(false); idFt.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY); final FieldType ft2 = new FieldType(); ft2.setIndexed(false); ft2.setStored(true); // model how solr does locking - only allow one thread to do a hard commit at once, and only one thread to do a soft commit, but // a hard commit in progress does not stop a soft commit. final Lock hardCommitLock = syncCommits ? new ReentrantLock() : null; final Lock reopenLock = syncCommits ? new ReentrantLock() : null; // RAMDirectory dir = new RAMDirectory(); // final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40))); Directory dir = newDirectory(); final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); writer.setDoRandomForceMergeAssert(false); // writer.commit(); // reader = IndexReader.open(dir); // make this reader an NRT reader from the start to avoid the first non-writer openIfChanged // to only opening at the last commit point. reader = DirectoryReader.open(writer.w, true); for (int i = 0; i < nWriteThreads; i++) { Thread thread = new Thread("WRITER" + i) { Random rand = new Random(random().nextInt()); @Override public void run() { try { while (operations.get() > 0) { int oper = rand.nextInt(100); if (oper < commitPercent) { if (numCommitting.incrementAndGet() <= maxConcurrentCommits) { Map<Integer, DocInfo> newCommittedModel; long version; DirectoryReader oldReader; boolean softCommit = rand.nextInt(100) < softCommitPercent; if (!softCommit) { // only allow one hard commit to proceed at once if (hardCommitLock != null) hardCommitLock.lock(); verbose("hardCommit start"); writer.commit(); } if (reopenLock != null) reopenLock.lock(); synchronized (globalLock) { newCommittedModel = new HashMap<Integer, DocInfo>(model); // take a snapshot version = snapshotCount++; oldReader = reader; oldReader.incRef(); // increment the reference since we will use this for reopening } if (!softCommit) { // must commit after taking a snapshot of the model // writer.commit(); } verbose("reopen start using", oldReader); DirectoryReader newReader; if (softCommit) { newReader = DirectoryReader.openIfChanged(oldReader, writer.w, true); } else { // will only open to last commit newReader = DirectoryReader.openIfChanged(oldReader); } if (newReader == null) { oldReader.incRef(); newReader = oldReader; } oldReader.decRef(); verbose("reopen result", newReader); synchronized (globalLock) { assert newReader.getRefCount() > 0; assert reader.getRefCount() > 0; // install the new reader if it's newest (and check the current version since another reader may have already been installed) if (newReader.getVersion() > reader.getVersion()) { reader.decRef(); reader = newReader; // install this snapshot only if it's newer than the current one if (version >= committedModelClock) { committedModel = newCommittedModel; committedModelClock = version; } } else { // close if unused newReader.decRef(); } } if (reopenLock != null) reopenLock.unlock(); if (!softCommit) { if (hardCommitLock != null) hardCommitLock.unlock(); } } numCommitting.decrementAndGet(); continue; } int id = rand.nextInt(ndocs); Object sync = syncArr[id]; // set the lastId before we actually change it sometimes to try and // uncover more race conditions between writing and reading boolean before = rand.nextBoolean(); if (before) { lastId = id; } // We can't concurrently update the same document and retain our invariants of increasing values // since we can't guarantee what order the updates will be executed. synchronized (sync) { DocInfo info = model.get(id); long val = info.val; long nextVal = Math.abs(val) + 1; if (oper < commitPercent + deletePercent) { // add tombstone first if (tombstones) { Document d = new Document(); d.add(new Field("id", "-" + Integer.toString(id), idFt)); d.add(new Field(field, Long.toString(nextVal), ft2)); verbose("adding tombstone for id", id, "val=", nextVal); writer.updateDocument(new Term("id", "-" + Integer.toString(id)), d); } verbose("deleting id", id, "val=", nextVal); writer.deleteDocuments(new Term("id", Integer.toString(id))); model.put(id, new DocInfo(0, -nextVal)); verbose("deleting id", id, "val=", nextVal, "DONE"); } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) { //assertU("<delete><query>id:" + id + "</query></delete>"); // add tombstone first if (tombstones) { Document d = new Document(); d.add(new Field("id", "-" + Integer.toString(id), idFt)); d.add(new Field(field, Long.toString(nextVal), ft2)); verbose("adding tombstone for id", id, "val=", nextVal); writer.updateDocument(new Term("id", "-" + Integer.toString(id)), d); } verbose("deleteByQuery", id, "val=", nextVal); writer.deleteDocuments(new TermQuery(new Term("id", Integer.toString(id)))); model.put(id, new DocInfo(0, -nextVal)); verbose("deleteByQuery", id, "val=", nextVal, "DONE"); } else { // model.put(id, nextVal); // uncomment this and this test should fail. // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); Document d = new Document(); d.add(new Field("id", Integer.toString(id), idFt)); d.add(new Field(field, Long.toString(nextVal), ft2)); verbose("adding id", id, "val=", nextVal); writer.updateDocument(new Term("id", Integer.toString(id)), d); if (tombstones) { // remove tombstone after new addition (this should be optional?) verbose("deleting tombstone for id", id); writer.deleteDocuments(new Term("id", "-" + Integer.toString(id))); verbose("deleting tombstone for id", id, "DONE"); } model.put(id, new DocInfo(0, nextVal)); verbose("adding id", id, "val=", nextVal, "DONE"); } } if (!before) { lastId = id; } } } catch (Exception ex) { throw new RuntimeException(ex); } } }; threads.add(thread); } for (int i = 0; i < nReadThreads; i++) { Thread thread = new Thread("READER" + i) { Random rand = new Random(random().nextInt()); @Override public void run() { try { while (operations.decrementAndGet() >= 0) { // bias toward a recently changed doc int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); // when indexing, we update the index, then the model // so when querying, we should first check the model, and then the index DocInfo info; synchronized (globalLock) { info = committedModel.get(id); } long val = info.val; IndexReader r; synchronized (globalLock) { r = reader; r.incRef(); } int docid = getFirstMatch(r, new Term("id", Integer.toString(id))); if (docid < 0 && tombstones) { // if we couldn't find the doc, look for it's tombstone docid = getFirstMatch(r, new Term("id", "-" + Integer.toString(id))); if (docid < 0) { if (val == -1L) { // expected... no doc was added yet r.decRef(); continue; } verbose("ERROR: Couldn't find a doc or tombstone for id", id, "using reader", r, "expected value", val); fail("No documents or tombstones found for id " + id + ", expected at least " + val); } } if (docid < 0 && !tombstones) { // nothing to do - we can't tell anything from a deleted doc without tombstones } else { if (docid < 0) { verbose("ERROR: Couldn't find a doc for id", id, "using reader", r); } assertTrue(docid >= 0); // we should have found the document, or it's tombstone Document doc = r.document(docid); long foundVal = Long.parseLong(doc.get(field)); if (foundVal < Math.abs(val)) { verbose("ERROR: id", id, "model_val=", val, " foundVal=", foundVal, "reader=", reader); } assertTrue(foundVal >= Math.abs(val)); } r.decRef(); } } catch (Throwable e) { operations.set(-1L); throw new RuntimeException(e); } } }; threads.add(thread); } for (Thread thread : threads) { thread.start(); } for (Thread thread : threads) { thread.join(); } writer.close(); reader.close(); dir.close(); }