List of usage examples for org.apache.lucene.index IndexWriter commit
@Override public final long commit() throws IOException
Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.
From source file:org.apache.solr.search.TestSort.java
License:Apache License
public void testSort() throws Exception { Directory dir = new RAMDirectory(); Field f = new StringField("f", "0", Field.Store.NO); Field f2 = new StringField("f2", "0", Field.Store.NO); for (int iterCnt = 0; iterCnt < iter; iterCnt++) { IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new SimpleAnalyzer(TEST_VERSION_CURRENT)) .setOpenMode(IndexWriterConfig.OpenMode.CREATE)); final MyDoc[] mydocs = new MyDoc[ndocs]; int v1EmptyPercent = 50; int v2EmptyPercent = 50; int commitCountdown = commitCount; for (int i = 0; i < ndocs; i++) { MyDoc mydoc = new MyDoc(); mydoc.doc = i;//w w w. j a va 2 s. com mydocs[i] = mydoc; Document document = new Document(); if (r.nextInt(100) < v1EmptyPercent) { mydoc.val = Integer.toString(r.nextInt(maxval)); f.setStringValue(mydoc.val); document.add(f); } if (r.nextInt(100) < v2EmptyPercent) { mydoc.val2 = Integer.toString(r.nextInt(maxval)); f2.setStringValue(mydoc.val2); document.add(f2); } iw.addDocument(document); if (--commitCountdown <= 0) { commitCountdown = commitCount; iw.commit(); } } iw.close(); DirectoryReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); // System.out.println("segments="+searcher.getIndexReader().getSequentialSubReaders().length); assertTrue(reader.leaves().size() > 1); for (int i = 0; i < qiter; i++) { Filter filt = new Filter() { @Override public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) { return BitsFilteredDocIdSet.wrap(randSet(context.reader().maxDoc()), acceptDocs); } }; int top = r.nextInt((ndocs >> 3) + 1) + 1; final boolean luceneSort = r.nextBoolean(); final boolean sortMissingLast = !luceneSort && r.nextBoolean(); final boolean sortMissingFirst = !luceneSort && !sortMissingLast; final boolean reverse = r.nextBoolean(); List<SortField> sfields = new ArrayList<SortField>(); final boolean secondary = r.nextBoolean(); final boolean luceneSort2 = r.nextBoolean(); final boolean sortMissingLast2 = !luceneSort2 && r.nextBoolean(); final boolean sortMissingFirst2 = !luceneSort2 && !sortMissingLast2; final boolean reverse2 = r.nextBoolean(); if (r.nextBoolean()) sfields.add(new SortField(null, SortField.Type.SCORE)); // hit both use-cases of sort-missing-last sfields.add(Sorting.getStringSortField("f", reverse, sortMissingLast, sortMissingFirst)); if (secondary) { sfields.add(Sorting.getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2)); } if (r.nextBoolean()) sfields.add(new SortField(null, SortField.Type.SCORE)); Sort sort = new Sort(sfields.toArray(new SortField[sfields.size()])); final String nullRep = luceneSort || sortMissingFirst && !reverse || sortMissingLast && reverse ? "" : "zzz"; final String nullRep2 = luceneSort2 || sortMissingFirst2 && !reverse2 || sortMissingLast2 && reverse2 ? "" : "zzz"; boolean trackScores = r.nextBoolean(); boolean trackMaxScores = r.nextBoolean(); boolean scoreInOrder = r.nextBoolean(); final TopFieldCollector topCollector = TopFieldCollector.create(sort, top, true, trackScores, trackMaxScores, scoreInOrder); final List<MyDoc> collectedDocs = new ArrayList<MyDoc>(); // delegate and collect docs ourselves Collector myCollector = new Collector() { int docBase; @Override public void setScorer(Scorer scorer) throws IOException { topCollector.setScorer(scorer); } @Override public void collect(int doc) throws IOException { topCollector.collect(doc); collectedDocs.add(mydocs[doc + docBase]); } @Override public void setNextReader(AtomicReaderContext context) throws IOException { topCollector.setNextReader(context); docBase = context.docBase; } @Override public boolean acceptsDocsOutOfOrder() { return topCollector.acceptsDocsOutOfOrder(); } }; searcher.search(new MatchAllDocsQuery(), filt, myCollector); Collections.sort(collectedDocs, new Comparator<MyDoc>() { @Override public int compare(MyDoc o1, MyDoc o2) { String v1 = o1.val == null ? nullRep : o1.val; String v2 = o2.val == null ? nullRep : o2.val; int cmp = v1.compareTo(v2); if (reverse) cmp = -cmp; if (cmp != 0) return cmp; if (secondary) { v1 = o1.val2 == null ? nullRep2 : o1.val2; v2 = o2.val2 == null ? nullRep2 : o2.val2; cmp = v1.compareTo(v2); if (reverse2) cmp = -cmp; } cmp = cmp == 0 ? o1.doc - o2.doc : cmp; return cmp; } }); TopDocs topDocs = topCollector.topDocs(); ScoreDoc[] sdocs = topDocs.scoreDocs; for (int j = 0; j < sdocs.length; j++) { int id = sdocs[j].doc; if (id != collectedDocs.get(j).doc) { log.error("Error at pos " + j + "\n\tsortMissingFirst=" + sortMissingFirst + " sortMissingLast=" + sortMissingLast + " reverse=" + reverse + "\n\tEXPECTED=" + collectedDocs); } assertEquals(id, collectedDocs.get(j).doc); } } reader.close(); } dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCacheReopen.java
License:Apache License
public void testFieldCacheReuseAfterReopen() throws Exception { Directory dir = newDirectory();/*w ww. j a v a 2s .c o m*/ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(10))); Document doc = new Document(); doc.add(new IntPoint("number", 17)); writer.addDocument(doc); writer.commit(); // Open reader1 DirectoryReader r = DirectoryReader.open(dir); LeafReader r1 = getOnlyLeafReader(r); final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER); assertEquals(0, ints.nextDoc()); assertEquals(17, ints.longValue()); // Add new segment writer.addDocument(doc); writer.commit(); // Reopen reader1 --> reader2 DirectoryReader r2 = DirectoryReader.openIfChanged(r); assertNotNull(r2); r.close(); LeafReader sub0 = r2.leaves().get(0).reader(); final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.INT_POINT_PARSER); r2.close(); assertEquals(0, ints2.nextDoc()); assertEquals(17, ints2.longValue()); writer.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCacheSort.java
License:Apache License
public void testEmptyStringVsNullStringSort() throws Exception { Directory dir = newDirectory();/* w w w. j a v a2 s.c o m*/ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); Document doc = new Document(); doc.add(newStringField("f", "", Field.Store.NO)); doc.add(newStringField("t", "1", Field.Store.NO)); w.addDocument(doc); w.commit(); doc = new Document(); doc.add(newStringField("t", "1", Field.Store.NO)); w.addDocument(doc); IndexReader r = UninvertingReader.wrap(DirectoryReader.open(w), Collections.singletonMap("f", Type.SORTED)); w.close(); IndexSearcher s = newSearcher(r); TopDocs hits = s.search(new TermQuery(new Term("t", "1")), 10, new Sort(new SortField("f", SortField.Type.STRING))); assertEquals(2, hits.totalHits); // null sorts first assertEquals(1, hits.scoreDocs[0].doc); assertEquals(0, hits.scoreDocs[1].doc); TestUtil.checkReader(r); r.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCacheSort.java
License:Apache License
public void testMaxScore() throws Exception { Directory d = newDirectory();/*from w ww . j av a 2 s. c om*/ // Not RIW because we need exactly 2 segs: IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random()))); int id = 0; for (int seg = 0; seg < 2; seg++) { for (int docIDX = 0; docIDX < 10; docIDX++) { Document doc = new Document(); doc.add(new LegacyIntField("id", docIDX, Field.Store.YES)); StringBuilder sb = new StringBuilder(); for (int i = 0; i < id; i++) { sb.append(' '); sb.append("text"); } doc.add(newTextField("body", sb.toString(), Field.Store.NO)); w.addDocument(doc); id++; } w.commit(); } IndexReader r = UninvertingReader.wrap(DirectoryReader.open(w), Collections.singletonMap("id", Type.LEGACY_INTEGER)); w.close(); Query q = new TermQuery(new Term("body", "text")); IndexSearcher s = newSearcher(r); float maxScore = s.search(q, 10).getMaxScore(); assertEquals(maxScore, s.search(q, 3, Sort.INDEXORDER, random().nextBoolean(), true).getMaxScore(), 0.0); assertEquals(maxScore, s.search(q, 3, Sort.RELEVANCE, random().nextBoolean(), true).getMaxScore(), 0.0); assertEquals(maxScore, s.search(q, 3, new Sort(new SortField[] { new SortField("id", SortField.Type.INT, false) }), random().nextBoolean(), true).getMaxScore(), 0.0); assertEquals(maxScore, s.search(q, 3, new Sort(new SortField[] { new SortField("id", SortField.Type.INT, true) }), random().nextBoolean(), true).getMaxScore(), 0.0); TestUtil.checkReader(r); r.close(); d.close(); }
From source file:org.apache.solr.update.DirectUpdateHandler2.java
License:Apache License
@Override public void commit(CommitUpdateCommand cmd) throws IOException { if (cmd.prepareCommit) { prepareCommit(cmd);// w w w .j ava2 s . co m return; } if (cmd.optimize) { optimizeCommands.incrementAndGet(); } else { commitCommands.incrementAndGet(); if (cmd.expungeDeletes) expungeDeleteCommands.incrementAndGet(); } Future[] waitSearcher = null; if (cmd.waitSearcher) { waitSearcher = new Future[1]; } boolean error = true; try { // only allow one hard commit to proceed at once if (!cmd.softCommit) { solrCoreState.getCommitLock().lock(); } log.info("start " + cmd); // We must cancel pending commits *before* we actually execute the commit. if (cmd.openSearcher) { // we can cancel any pending soft commits if this commit will open a new searcher softCommitTracker.cancelPendingCommit(); } if (!cmd.softCommit && (cmd.openSearcher || !commitTracker.getOpenSearcher())) { // cancel a pending hard commit if this commit is of equal or greater "strength"... // If the autoCommit has openSearcher=true, then this commit must have openSearcher=true // to cancel. commitTracker.cancelPendingCommit(); } RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core); try { IndexWriter writer = iw.get(); if (cmd.optimize) { writer.forceMerge(cmd.maxOptimizeSegments); } else if (cmd.expungeDeletes) { writer.forceMergeDeletes(); } if (!cmd.softCommit) { synchronized (solrCoreState.getUpdateLock()) { // sync is currently needed to prevent preCommit // from being called between preSoft and // postSoft... see postSoft comments. if (ulog != null) ulog.preCommit(cmd); } // SolrCore.verbose("writer.commit() start writer=",writer); if (writer.hasUncommittedChanges()) { final Map<String, String> commitData = new HashMap<String, String>(); commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis())); writer.setCommitData(commitData); writer.commit(); } else { log.info("No uncommitted changes. Skipping IW.commit."); } // SolrCore.verbose("writer.commit() end"); numDocsPending.set(0); callPostCommitCallbacks(); } else { callPostSoftCommitCallbacks(); } } finally { iw.decref(); } if (cmd.optimize) { callPostOptimizeCallbacks(); } if (cmd.softCommit) { // ulog.preSoftCommit(); synchronized (solrCoreState.getUpdateLock()) { if (ulog != null) ulog.preSoftCommit(cmd); core.getSearcher(true, false, waitSearcher, true); if (ulog != null) ulog.postSoftCommit(cmd); } // ulog.postSoftCommit(); } else { synchronized (solrCoreState.getUpdateLock()) { if (ulog != null) ulog.preSoftCommit(cmd); if (cmd.openSearcher) { core.getSearcher(true, false, waitSearcher); } else { // force open a new realtime searcher so realtime-get and versioning code can see the latest RefCounted<SolrIndexSearcher> searchHolder = core.openNewSearcher(true, true); searchHolder.decref(); } if (ulog != null) ulog.postSoftCommit(cmd); } if (ulog != null) ulog.postCommit(cmd); // postCommit currently means new searcher has // also been opened } // reset commit tracking if (cmd.softCommit) { softCommitTracker.didCommit(); } else { commitTracker.didCommit(); } log.info("end_commit_flush"); error = false; } finally { if (!cmd.softCommit) { solrCoreState.getCommitLock().unlock(); } addCommands.set(0); deleteByIdCommands.set(0); deleteByQueryCommands.set(0); if (error) numErrors.incrementAndGet(); } // if we are supposed to wait for the searcher to be registered, then we should do it // outside any synchronized block so that other update operations can proceed. if (waitSearcher != null && waitSearcher[0] != null) { try { waitSearcher[0].get(); } catch (InterruptedException e) { SolrException.log(log, e); } catch (ExecutionException e) { SolrException.log(log, e); } } }
From source file:org.apache.solr.update.DirectUpdateHandler2.java
License:Apache License
@Override public void closeWriter(IndexWriter writer) throws IOException { boolean clearRequestInfo = false; solrCoreState.getCommitLock().lock(); try {/* www . ja va 2 s.com*/ SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams()); SolrQueryResponse rsp = new SolrQueryResponse(); if (SolrRequestInfo.getRequestInfo() == null) { clearRequestInfo = true; SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); // important for debugging } if (!commitOnClose) { if (writer != null) { writer.rollback(); } // we shouldn't close the transaction logs either, but leaving them open // means we can't delete them on windows (needed for tests) if (ulog != null) ulog.close(false); return; } // do a commit before we quit? boolean tryToCommit = writer != null && ulog != null && ulog.hasUncommittedChanges() && ulog.getState() == UpdateLog.State.ACTIVE; try { if (tryToCommit) { CommitUpdateCommand cmd = new CommitUpdateCommand(req, false); cmd.openSearcher = false; cmd.waitSearcher = false; cmd.softCommit = false; // TODO: keep other commit callbacks from being called? // this.commit(cmd); // too many test failures using this method... is it because of callbacks? synchronized (solrCoreState.getUpdateLock()) { ulog.preCommit(cmd); } // todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't be used) final Map<String, String> commitData = new HashMap<String, String>(); commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis())); writer.setCommitData(commitData); writer.commit(); synchronized (solrCoreState.getUpdateLock()) { ulog.postCommit(cmd); } } } catch (Throwable th) { log.error("Error in final commit", th); } // we went through the normal process to commit, so we don't have to artificially // cap any ulog files. try { if (ulog != null) ulog.close(false); } catch (Throwable th) { log.error("Error closing log files", th); } if (writer != null) writer.close(); } finally { solrCoreState.getCommitLock().unlock(); if (clearRequestInfo) SolrRequestInfo.clearRequestInfo(); } }
From source file:org.arastreju.sge.spi.impl.LuceneBasedNodeKeyTable.java
License:Apache License
@Override public synchronized void put(QualifiedName qn, T physicalID) { Document doc = new Document(); doc.add(new Field(QN, qn.toURI(), Field.Store.YES, Field.Index.NOT_ANALYZED)); setID(doc, physicalID);// w w w . jav a2s. co m try { final IndexWriter writer = writer(); writer.addDocument(doc); writer.commit(); } catch (IOException e) { throw new RuntimeException("Could not map phyiscal ID to qualified name in index.", e); } }
From source file:org.arastreju.sge.spi.impl.LuceneBasedNodeKeyTable.java
License:Apache License
@Override public void remove(QualifiedName qn) { try {/*from www .ja v a 2s . c o m*/ final IndexWriter writer = writer(); writer.deleteDocuments(new Term(QN, qn.toURI())); writer.commit(); } catch (IOException e) { throw new RuntimeException("Could not remove qualified name from index.", e); } }
From source file:org.archive.index.IndexFiles.java
License:Apache License
/** * index files: ..._check.xml/* www .j av a 2s . co m*/ * **/ public static void indexFiles_check(String dirStr) { String indexPath = TDirectory.LPFileIndexPath; try { logOddFile = new PrintStream( new FileOutputStream(new File(TDirectory.ROOT_OUTPUT + "logOddCheckIndexFiles.txt"))); System.out.println("Indexing to directory '" + indexPath + "'..."); Directory dir = FSDirectory.open(new File(indexPath)); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_48); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_48, analyzer); boolean create = true; if (create) { // Create a new index in the directory, removing any previously indexed documents: iwc.setOpenMode(OpenMode.CREATE); } else { // Add new documents to an existing index: iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); } IndexWriter indexWriter = new IndexWriter(dir, iwc); Date start = new Date(); File dirFile = new File(dirStr); File[] files = dirFile.listFiles(); System.out.println(files.length); int count = 1; int badCount = 0; for (File f : files) { System.out.print("file-" + count + "\t"); count++; List<org.apache.lucene.document.Document> docs = new ArrayList<org.apache.lucene.document.Document>(); List<TreeMap<String, String>> checkdocList = TemLoader.parseCheckFile(logOddFile, f.getAbsolutePath()); if (null == checkdocList) { System.out.print("null"); System.out.println(); badCount++; continue; } System.out.print(f); System.out.println(); for (TreeMap<String, String> checkdoc : checkdocList) { // make a new, empty document org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document(); Field idField = new StringField("id", checkdoc.get("id"), Field.Store.YES); doc.add(idField); for (Entry<String, String> entry : checkdoc.entrySet()) { if (!entry.getKey().equals("id")) { StoredField storeField = new StoredField(entry.getKey(), entry.getValue()); doc.add(storeField); } } docs.add(doc); } for (org.apache.lucene.document.Document doc : docs) { indexWriter.addDocument(doc); } } indexWriter.commit(); indexWriter.close(); logOddFile.flush(); logOddFile.close(); Date end = new Date(); System.out.println(end.getTime() - start.getTime() + " total milliseconds"); System.out.println("BadCount:\t" + badCount); } catch (Exception e) { // TODO: handle exception e.printStackTrace(); } }
From source file:org.archive.tnh.tools.IndexMerger.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length < 2) { System.err.println("IndexMerger [-v|-o|-f] <dest> <source>..."); System.exit(1);//from www .jav a2 s .co m } boolean verbose = false; boolean optimize = false; boolean force = false; int i = 0; for (; i < args.length; i++) { if ("-o".equals(args[i])) { optimize = true; } else if ("-f".equals(args[i])) { force = true; } else if ("-v".equals(args[i])) { verbose = true; } else { break; } } if ((args.length - i) < (2 - (optimize ? 1 : 0))) { System.err.println("Erorr: no source files!"); System.err.println("IndexMerger [-v|-o|-f] <dest> <source>..."); System.exit(1); } File dest = new File(args[i++]); if (!force && dest.exists()) { System.err.println("Destination exits, use -f to force merging into existing index: " + dest); System.exit(2); } IndexReader ir[] = new IndexReader[args.length - i]; for (int j = i; j < args.length; j++) { ir[j - i] = IndexReader.open(new MMapDirectory(new File(args[j])), true /* read-only */ ); } IndexWriter w = null; try { // Configure the IndexWriter. IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35, null); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); w = new IndexWriter(new MMapDirectory(dest), config); if (verbose) { w.setInfoStream(System.out); } if (ir.length > 0) { w.addIndexes(ir); } if (optimize) { w.optimize(); } w.commit(); w.close(); } catch (IOException ioe) { System.err.println("Error: " + args[0] + " " + ioe); if (w != null) w.close(); } }