List of usage examples for org.apache.lucene.index IndexWriter commit
@Override public final long commit() throws IOException
Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.
From source file:org.elasticsearch.index.engine.EngineDiskUtils.java
License:Apache License
private static void updateCommitData(IndexWriter writer, Map<String, String> keysToUpdate) throws IOException { final Map<String, String> userData = getUserData(writer); userData.putAll(keysToUpdate);/*from w w w . ja v a 2s .c om*/ writer.setLiveCommitData(userData.entrySet()); writer.commit(); }
From source file:org.elasticsearch.index.engine.internal.AsynchronousEngine.java
License:Apache License
@Override public void flush(Flush flush) throws EngineException { ensureOpen();//w w w .j a v a 2s .co m if (flush.type() == Flush.Type.NEW_WRITER || flush.type() == Flush.Type.COMMIT_TRANSLOG) { // check outside the lock as well so we can check without blocking on the write lock if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "recovery is in progress, flush [" + flush.type() + "] is not allowed"); } } int currentFlushing = flushing.incrementAndGet(); if (currentFlushing > 1 && !flush.waitIfOngoing()) { flushing.decrementAndGet(); throw new FlushNotAllowedEngineException(shardId, "already flushing..."); } flushLock.lock(); try { if (flush.type() == Flush.Type.NEW_WRITER) { try (InternalLock _ = writeLock.acquire()) { if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "Recovery is in progress, flush is not allowed"); } // disable refreshing, not dirty dirty = false; try { { // commit and close the current writer - we write the current tanslog ID just in case final long translogId = translog.currentId(); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); indexWriter.rollback(); } indexWriter = createWriter(); mergeScheduler.removeListener(this.throttle); this.throttle = new IndexThrottle(mergeScheduler, this.logger, indexingService); mergeScheduler.addListener(throttle); // commit on a just opened writer will commit even if there are no changes done to it // we rely on that for the commit data translog id key if (flushNeeded || flush.force()) { flushNeeded = false; long translogId = translogIdGenerator.incrementAndGet(); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); translog.newTranslog(translogId); } SearcherManager current = this.searcherManager; this.searcherManager = buildSearchManager(indexWriter); versionMap.setManager(searcherManager); try { IOUtils.close(current); } catch (Throwable t) { logger.warn("Failed to close current SearcherManager", t); } maybePruneDeletedTombstones(); } catch (Throwable t) { throw new FlushFailedEngineException(shardId, t); } } } else if (flush.type() == Flush.Type.COMMIT_TRANSLOG) { try (InternalLock _ = readLock.acquire()) { final IndexWriter indexWriter = currentIndexWriter(); if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "Recovery is in progress, flush is not allowed"); } if (flushNeeded || flush.force()) { flushNeeded = false; try { long translogId = translogIdGenerator.incrementAndGet(); translog.newTransientTranslog(translogId); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); // we need to refresh in order to clear older version values refresh(new Refresh("version_table_flush").force(true)); // we need to move transient to current only after we refresh // so items added to current will still be around for realtime get // when tans overrides it translog.makeTransientCurrent(); } catch (Throwable e) { translog.revertTransient(); throw new FlushFailedEngineException(shardId, e); } } } // We don't have to do this here; we do it defensively to make sure that even if wall clock time is misbehaving // (e.g., moves backwards) we will at least still sometimes prune deleted tombstones: if (enableGcDeletes) { pruneDeletedTombstones(); } } else if (flush.type() == Flush.Type.COMMIT) { // note, its ok to just commit without cleaning the translog, its perfectly fine to replay a // translog on an index that was opened on a committed point in time that is "in the future" // of that translog try (InternalLock _ = readLock.acquire()) { final IndexWriter indexWriter = currentIndexWriter(); // we allow to *just* commit if there is an ongoing recovery happening... // its ok to use this, only a flush will cause a new translogId, and we are locked here from // other flushes use flushLock try { long translogId = translog.currentId(); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); } catch (Throwable e) { throw new FlushFailedEngineException(shardId, e); } } // We don't have to do this here; we do it defensively to make sure that even if wall clock time is misbehaving // (e.g., moves backwards) we will at least still sometimes prune deleted tombstones: if (enableGcDeletes) { pruneDeletedTombstones(); } } else { throw new ElasticsearchIllegalStateException("flush type [" + flush.type() + "] not supported"); } // reread the last committed segment infos try (InternalLock _ = readLock.acquire()) { ensureOpen(); readLastCommittedSegmentsInfo(); } catch (Throwable e) { if (!closed) { logger.warn("failed to read latest segment infos on flush", e); if (Lucene.isCorruptionException(e)) { throw new FlushFailedEngineException(shardId, e); } } } } catch (FlushFailedEngineException ex) { maybeFailEngine(ex, "flush"); throw ex; } finally { flushLock.unlock(); flushing.decrementAndGet(); } }
From source file:org.elasticsearch.index.engine.internal.ZyucInternalEngine.java
License:Apache License
@Override public void flush(Flush flush) throws EngineException { ensureOpen();//w w w. j a v a2 s. c om if (flush.type() == Flush.Type.NEW_WRITER || flush.type() == Flush.Type.COMMIT_TRANSLOG) { // check outside the lock as well so we can check without blocking on the write lock if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "recovery is in progress, flush [" + flush.type() + "] is not allowed"); } } int currentFlushing = flushing.incrementAndGet(); if (currentFlushing > 1 && !flush.waitIfOngoing()) { flushing.decrementAndGet(); throw new FlushNotAllowedEngineException(shardId, "already flushing..."); } flushLock.lock(); try { if (flush.type() == Flush.Type.NEW_WRITER) { try (InternalLock _ = writeLock.acquire()) { if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "Recovery is in progress, flush is not allowed"); } // disable refreshing, not dirty dirty = false; try { { // commit and close the current writer - we write the current tanslog ID just in case final long translogId = translog.currentId(); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); indexWriter.rollback(); } indexWriter = createWriter(); mergeScheduler.removeListener(this.throttle); this.throttle = new IndexThrottle(mergeScheduler, this.logger, indexingService); mergeScheduler.addListener(throttle); // commit on a just opened writer will commit even if there are no changes done to it // we rely on that for the commit data translog id key if (flushNeeded || flush.force()) { flushNeeded = false; long translogId = translogIdGenerator.incrementAndGet(); indexWriter.setCommitData( Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); translog.newTranslog(translogId); } SearcherManager current = this.searcherManager; this.searcherManager = buildSearchManager(indexWriter); versionMap.setManager(searcherManager); try { IOUtils.close(current); } catch (Throwable t) { logger.warn("Failed to close current SearcherManager", t); } maybePruneDeletedTombstones(); } catch (Throwable t) { throw new FlushFailedEngineException(shardId, t); } } } else if (flush.type() == Flush.Type.COMMIT_TRANSLOG) { try (InternalLock _ = readLock.acquire()) { final IndexWriter indexWriter = currentIndexWriter(); if (onGoingRecoveries.get() > 0) { throw new FlushNotAllowedEngineException(shardId, "Recovery is in progress, flush is not allowed"); } if (flushNeeded || flush.force()) { flushNeeded = false; try { long translogId = translogIdGenerator.incrementAndGet(); translog.newTransientTranslog(translogId); // indexWriter.setCommitData(Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); // we need to refresh in order to clear older version values refresh(new Refresh("version_table_flush").force(true)); // we need to move transient to current only after we refresh // so items added to current will still be around for realtime get // when tans overrides it translog.makeTransientCurrent(); } catch (Throwable e) { translog.revertTransient(); throw new FlushFailedEngineException(shardId, e); } } } // We don't have to do this here; we do it defensively to make sure that even if wall clock time is misbehaving // (e.g., moves backwards) we will at least still sometimes prune deleted tombstones: if (enableGcDeletes) { pruneDeletedTombstones(); } } else if (flush.type() == Flush.Type.COMMIT) { // note, its ok to just commit without cleaning the translog, its perfectly fine to replay a // translog on an index that was opened on a committed point in time that is "in the future" // of that translog try (InternalLock _ = readLock.acquire()) { final IndexWriter indexWriter = currentIndexWriter(); // we allow to *just* commit if there is an ongoing recovery happening... // its ok to use this, only a flush will cause a new translogId, and we are locked here from // other flushes use flushLock try { long translogId = translog.currentId(); // indexWriter.setCommitData(Collections.singletonMap(Translog.TRANSLOG_ID_KEY, Long.toString(translogId))); indexWriter.commit(); } catch (Throwable e) { throw new FlushFailedEngineException(shardId, e); } } // We don't have to do this here; we do it defensively to make sure that even if wall clock time is misbehaving // (e.g., moves backwards) we will at least still sometimes prune deleted tombstones: if (enableGcDeletes) { pruneDeletedTombstones(); } } else { throw new ElasticsearchIllegalStateException("flush type [" + flush.type() + "] not supported"); } // reread the last committed segment infos try (InternalLock _ = readLock.acquire()) { ensureOpen(); readLastCommittedSegmentsInfo(); } catch (Throwable e) { if (!closed) { logger.warn("failed to read latest segment infos on flush", e); if (Lucene.isCorruptionException(e)) { throw new FlushFailedEngineException(shardId, e); } } } } catch (FlushFailedEngineException ex) { maybeFailEngine(ex, "flush"); throw ex; } finally { flushLock.unlock(); flushing.decrementAndGet(); } }
From source file:org.elasticsearch.index.engine.InternalEngine.java
License:Apache License
private void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException { try {/* www . j a v a 2 s . co m*/ /* Translog.TranslogGeneration translogGeneration = translog.getGeneration(); logger.trace("committing writer with translog id [{}] and sync id [{}] ", translogGeneration.translogFileGeneration, syncId); Map<String, String> commitData = new HashMap<>(2); commitData.put(Translog.TRANSLOG_GENERATION_KEY, Long.toString(translogGeneration.translogFileGeneration)); commitData.put(Translog.TRANSLOG_UUID_KEY, translogGeneration.translogUUID); if (syncId != null) { commitData.put(Engine.SYNC_COMMIT_ID, syncId); } indexWriter.setCommitData(commitData); */ writer.commit(); } catch (Throwable ex) { failEngine("lucene commit failed", ex); throw ex; } }
From source file:org.elasticsearch.index.fielddata.FieldDataCacheTests.java
License:Apache License
public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception { Directory dir = newDirectory();/*from www. jav a 2s. c o m*/ IndexWriterConfig iwc = new IndexWriterConfig(null); iwc.setMergePolicy(NoMergePolicy.INSTANCE); IndexWriter iw = new IndexWriter(dir, iwc); long numDocs = scaledRandomIntBetween(32, 128); for (int i = 1; i <= numDocs; i++) { Document doc = new Document(); doc.add(new SortedSetDocValuesField("field1", new BytesRef(String.valueOf(i)))); doc.add(new StringField("field2", String.valueOf(i), Field.Store.NO)); iw.addDocument(doc); if (i % 24 == 0) { iw.commit(); } } iw.close(); DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", 0)); DummyAccountingFieldDataCache fieldDataCache = new DummyAccountingFieldDataCache(); // Testing SortedSetDVOrdinalsIndexFieldData: SortedSetDVOrdinalsIndexFieldData sortedSetDVOrdinalsIndexFieldData = createSortedDV("field1", fieldDataCache); sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir); assertThat(fieldDataCache.cachedGlobally, equalTo(1)); sortedSetDVOrdinalsIndexFieldData.loadGlobal(new FieldMaskingReader("field1", ir)); assertThat(fieldDataCache.cachedGlobally, equalTo(1)); // Testing PagedBytesIndexFieldData PagedBytesIndexFieldData pagedBytesIndexFieldData = createPagedBytes("field2", fieldDataCache); pagedBytesIndexFieldData.loadGlobal(ir); assertThat(fieldDataCache.cachedGlobally, equalTo(2)); pagedBytesIndexFieldData.loadGlobal(new FieldMaskingReader("field2", ir)); assertThat(fieldDataCache.cachedGlobally, equalTo(2)); ir.close(); dir.close(); }
From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java
License:Apache License
/** Simple test: one doc in the old format, check that it looks correct */ public void testUpgradeOneDocument() throws Exception { Directory dir = newDirectory();// w w w . j a v a 2 s . c om IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null)); // add a document with a _uid having a payload of 3 Document doc = new Document(); Token token = new Token("1", 0, 1); token.setPayload(new BytesRef(Numbers.longToBytes(3))); doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token))); iw.addDocument(doc); iw.commit(); CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true)); CodecReader upgraded = VersionFieldUpgrader.wrap(reader); // we need to be upgraded, should be a different instance assertNotSame(reader, upgraded); // make sure we can see our numericdocvalues in fieldinfos FieldInfo versionField = upgraded.getFieldInfos().fieldInfo(VersionFieldMapper.NAME); assertNotNull(versionField); assertEquals(DocValuesType.NUMERIC, versionField.getDocValuesType()); // should have a value of 3, and be visible in docsWithField assertEquals(3, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(0)); assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(0)); // verify filterreader with checkindex TestUtil.checkReader(upgraded); reader.close(); iw.close(); dir.close(); }
From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java
License:Apache License
/** test that we are a non-op if the segment already has the version field */ public void testAlreadyUpgraded() throws Exception { Directory dir = newDirectory();//from w w w .j a v a2 s. c o m IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null)); // add a document with a _uid having a payload of 3 Document doc = new Document(); Token token = new Token("1", 0, 1); token.setPayload(new BytesRef(Numbers.longToBytes(3))); doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token))); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 3)); iw.addDocument(doc); iw.commit(); CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true)); CodecReader upgraded = VersionFieldUpgrader.wrap(reader); // we already upgraded: should be same instance assertSame(reader, upgraded); reader.close(); iw.close(); dir.close(); }
From source file:org.elasticsearch.index.merge.policy.VersionFieldUpgraderTest.java
License:Apache License
/** Test upgrading two documents */ public void testUpgradeTwoDocuments() throws Exception { Directory dir = newDirectory();/*www.j a v a 2s.co m*/ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null)); // add a document with a _uid having a payload of 3 Document doc = new Document(); Token token = new Token("1", 0, 1); token.setPayload(new BytesRef(Numbers.longToBytes(3))); doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token))); iw.addDocument(doc); doc = new Document(); token = new Token("2", 0, 1); token.setPayload(new BytesRef(Numbers.longToBytes(4))); doc.add(new TextField(UidFieldMapper.NAME, new CannedTokenStream(token))); iw.addDocument(doc); iw.commit(); CodecReader reader = getOnlySegmentReader(DirectoryReader.open(iw, true)); CodecReader upgraded = VersionFieldUpgrader.wrap(reader); // we need to be upgraded, should be a different instance assertNotSame(reader, upgraded); // make sure we can see our numericdocvalues in fieldinfos FieldInfo versionField = upgraded.getFieldInfos().fieldInfo(VersionFieldMapper.NAME); assertNotNull(versionField); assertEquals(DocValuesType.NUMERIC, versionField.getDocValuesType()); // should have a values of 3 and 4, and be visible in docsWithField assertEquals(3, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(0)); assertEquals(4, upgraded.getNumericDocValues(VersionFieldMapper.NAME).get(1)); assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(0)); assertTrue(upgraded.getDocsWithField(VersionFieldMapper.NAME).get(1)); // verify filterreader with checkindex TestUtil.checkReader(upgraded); reader.close(); iw.close(); dir.close(); }
From source file:org.elasticsearch.index.shard.ShardUtilsTests.java
License:Apache License
public void testExtractShardId() throws IOException { BaseDirectoryWrapper dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); writer.commit(); ShardId id = new ShardId("foo", random().nextInt()); try (DirectoryReader reader = DirectoryReader.open(writer, random().nextBoolean())) { ElasticsearchDirectoryReader wrap = ElasticsearchDirectoryReader.wrap(reader, id); assertEquals(id, ShardUtils.extractShardId(wrap)); }/*from w w w . j ava 2 s . c om*/ final int numDocs = 1 + random().nextInt(5); for (int i = 0; i < numDocs; i++) { Document d = new Document(); d.add(newField("name", "foobar", StringField.TYPE_STORED)); writer.addDocument(d); if (random().nextBoolean()) { writer.commit(); } } try (DirectoryReader reader = DirectoryReader.open(writer, random().nextBoolean())) { ElasticsearchDirectoryReader wrap = ElasticsearchDirectoryReader.wrap(reader, id); assertEquals(id, ShardUtils.extractShardId(wrap)); CompositeReaderContext context = wrap.getContext(); for (LeafReaderContext leaf : context.leaves()) { assertEquals(id, ShardUtils.extractShardId(leaf.reader())); } } IOUtils.close(writer, dir); }
From source file:org.elasticsearch.index.shard.StoreRecoveryTests.java
License:Apache License
public void testAddIndices() throws IOException { Directory[] dirs = new Directory[randomIntBetween(1, 10)]; final int numDocs = randomIntBetween(50, 100); int id = 0;// ww w . j a v a 2s. c o m for (int i = 0; i < dirs.length; i++) { dirs[i] = newFSDirectory(createTempDir()); IndexWriter writer = new IndexWriter(dirs[i], newIndexWriterConfig() .setMergePolicy(NoMergePolicy.INSTANCE).setOpenMode(IndexWriterConfig.OpenMode.CREATE)); for (int j = 0; j < numDocs; j++) { writer.addDocument(Arrays.asList(new StringField("id", Integer.toString(id++), Field.Store.YES))); } writer.commit(); writer.close(); } StoreRecovery storeRecovery = new StoreRecovery(new ShardId("foo", "bar", 1), logger); RecoveryState.Index indexStats = new RecoveryState.Index(); Directory target = newFSDirectory(createTempDir()); storeRecovery.addIndices(indexStats, target, dirs); int numFiles = 0; Predicate<String> filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false && f.startsWith("extra") == false; for (Directory d : dirs) { numFiles += Arrays.asList(d.listAll()).stream().filter(filesFilter).count(); } final long targetNumFiles = Arrays.asList(target.listAll()).stream().filter(filesFilter).count(); assertEquals(numFiles, targetNumFiles); assertEquals(indexStats.totalFileCount(), targetNumFiles); if (hardLinksSupported(createTempDir())) { assertEquals(targetNumFiles, indexStats.reusedFileCount()); } else { assertEquals(0, indexStats.reusedFileCount(), 0); } DirectoryReader reader = DirectoryReader.open(target); SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target); for (SegmentCommitInfo info : segmentCommitInfos) { // check that we didn't merge assertEquals("all sources must be flush", info.info.getDiagnostics().get("source"), "flush"); } assertEquals(reader.numDeletedDocs(), 0); assertEquals(reader.numDocs(), id); reader.close(); target.close(); IOUtils.close(dirs); }