List of usage examples for org.apache.lucene.index DirectoryReader listCommits
public static List<IndexCommit> listCommits(Directory dir) throws IOException
From source file:com.github.rnewson.couchdb.lucene.DatabaseIndexer.java
License:Apache License
private UpdateSequence getUpdateSequence(final Directory dir) throws IOException { if (!DirectoryReader.indexExists(dir)) { return UpdateSequence.START; }/*ww w . j a v a 2 s . c o m*/ final List<IndexCommit> commits = DirectoryReader.listCommits(dir); final IndexCommit latest = commits.get(commits.size() - 1); return getUpdateSequence(latest.getUserData()); }
From source file:org.apache.blur.mapreduce.lib.BlurInputFormat.java
License:Apache License
public static List<BlurInputSplit> getSplitForDirectory(Path shardDir, Configuration configuration, Text table, Text snapshot, Directory directory) throws IOException { List<BlurInputSplit> splits = new ArrayList<BlurInputSplit>(); SnapshotIndexDeletionPolicy policy = new SnapshotIndexDeletionPolicy(configuration, SnapshotIndexDeletionPolicy.getGenerationsPath(shardDir)); Long generation = policy.getGeneration(snapshot.toString()); if (generation == null) { throw new IOException("Snapshot [" + snapshot + "] not found in shard [" + shardDir + "]"); }//from w ww . j a va2s.com List<IndexCommit> listCommits = DirectoryReader.listCommits(directory); IndexCommit indexCommit = findIndexCommit(listCommits, generation, shardDir); String segmentsFileName = indexCommit.getSegmentsFileName(); SegmentInfos segmentInfos = new SegmentInfos(); segmentInfos.read(directory, segmentsFileName); for (SegmentInfoPerCommit commit : segmentInfos) { SegmentInfo segmentInfo = commit.info; if (commit.getDelCount() == segmentInfo.getDocCount()) { LOG.info("Segment [{0}] in dir [{1}] has all records deleted.", segmentInfo.name, shardDir); } else { String name = segmentInfo.name; Collection<String> files = commit.files(); long fileLength = 0; for (String file : files) { fileLength += directory.fileLength(file); } List<String> dirFiles = new ArrayList<String>(files); dirFiles.add(segmentsFileName); splits.add(new BlurInputSplit(shardDir, segmentsFileName, name, fileLength, table, dirFiles)); } } return splits; }
From source file:org.apache.blur.store.hdfs_v2.FastHdfsKeyValueDirectoryTest.java
License:Apache License
@Test public void testMulipleCommitsAndReopens() throws IOException { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); conf.setMergeScheduler(new SerialMergeScheduler()); TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); Set<String> fileSet = new TreeSet<String>(); long seed = new Random().nextLong(); System.out.println("Seed:" + seed); Random random = new Random(seed); int docCount = 0; int passes = 10; byte[] segmentsGenContents = null; for (int run = 0; run < passes; run++) { final FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path, "test_multiple_commits_reopens")); if (segmentsGenContents != null) { byte[] segmentsGenContentsCurrent = readSegmentsGen(directory); assertTrue(Arrays.equals(segmentsGenContents, segmentsGenContentsCurrent)); }/*from w w w.j av a 2 s. co m*/ assertFiles(fileSet, run, -1, directory); assertEquals(docCount, getDocumentCount(directory)); IndexWriter writer = new IndexWriter(directory, conf.clone()); int numberOfCommits = random.nextInt(100); for (int i = 0; i < numberOfCommits; i++) { assertFiles(fileSet, run, i, directory); addDocuments(writer, random.nextInt(100)); // Before Commit writer.commit(); // After Commit // Set files after commit { fileSet.clear(); List<IndexCommit> listCommits = DirectoryReader.listCommits(directory); assertEquals(1, listCommits.size()); IndexCommit indexCommit = listCommits.get(0); fileSet.addAll(indexCommit.getFileNames()); } segmentsGenContents = readSegmentsGen(directory); } docCount = getDocumentCount(directory); } }
From source file:org.apache.blur.store.hdfs_v2.FastHdfsKeyValueDirectoryTest.java
License:Apache License
private void assertFiles(Set<String> expected, int run, int commit, FastHdfsKeyValueDirectory directory) throws IOException { Set<String> actual; if (DirectoryReader.indexExists(directory)) { List<IndexCommit> listCommits = DirectoryReader.listCommits(directory); // assertEquals(1, listCommits.size()); IndexCommit indexCommit = listCommits.get(0); actual = new TreeSet<String>(indexCommit.getFileNames()); } else {/* ww w . j a v a 2 s. c om*/ actual = new TreeSet<String>(); } Set<String> missing = new TreeSet<String>(expected); missing.removeAll(actual); Set<String> extra = new TreeSet<String>(actual); extra.removeAll(expected); assertEquals("Pass [" + run + "] Missing Files " + " Extra Files " + extra + "", expected, actual); }
From source file:org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.java
License:Apache License
/** * This method returns the {@linkplain IndexCommit} associated with the specified * <code>commitName</code>. A snapshot with specified <code>commitName</code> must * be created before invoking this method. * * @param commitName The name of persisted commit * @return the {@linkplain IndexCommit}/*from w ww. j a v a2s . c o m*/ * @throws IOException in case of I/O error. */ public Optional<IndexCommit> getIndexCommitByName(String commitName) throws IOException { Optional<IndexCommit> result = Optional.empty(); Optional<SnapshotMetaData> metaData = getSnapshotMetaData(commitName); if (metaData.isPresent()) { String indexDirPath = metaData.get().getIndexDirPath(); long gen = metaData.get().getGenerationNumber(); Directory d = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE); try { result = DirectoryReader.listCommits(d).stream().filter(ic -> ic.getGeneration() == gen).findAny(); if (!result.isPresent()) { log.warn("Unable to find commit with generation {} in the directory {}", gen, indexDirPath); } } finally { solrCore.getDirectoryFactory().release(d); } } else { log.warn("Commit with name {} is not persisted for core {}", commitName, solrCore.getName()); } return result; }
From source file:org.apache.solr.core.snapshots.TestSolrCoreSnapshots.java
License:Apache License
private List<IndexCommit> listCommits(String directory) throws Exception { SimpleFSDirectory dir = new SimpleFSDirectory(Paths.get(directory)); try {/*from ww w . jav a 2 s .c o m*/ return DirectoryReader.listCommits(dir); } catch (IndexNotFoundException ex) { // This can happen when the delete snapshot functionality cleans up the index files (when the directory // storing these files is not the *current* index directory). return Collections.emptyList(); } }
From source file:org.apache.solr.handler.ReplicationHandler.java
License:Apache License
@Override @SuppressWarnings("unchecked") public void inform(SolrCore core) { this.core = core; registerFileStreamResponseWriter();/* w w w . j ava2 s . c o m*/ registerCloseHook(); Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM); if (nbtk != null) { numberBackupsToKeep = Integer.parseInt(nbtk.toString()); } else { numberBackupsToKeep = 0; } NamedList slave = (NamedList) initArgs.get("slave"); boolean enableSlave = isEnabled(slave); if (enableSlave) { tempSnapPuller = snapPuller = new SnapPuller(slave, this, core); isSlave = true; } NamedList master = (NamedList) initArgs.get("master"); boolean enableMaster = isEnabled(master); if (enableMaster || enableSlave) { if (core.getCoreDescriptor().getCoreContainer().getZkController() != null) { LOG.warn("SolrCloud is enabled for core " + core.getName() + " but so is old-style replication. Make sure you" + " intend this behavior, it usually indicates a mis-configuration. Master setting is " + Boolean.toString(enableMaster) + " and slave setting is " + Boolean.toString(enableSlave)); } } if (!enableSlave && !enableMaster) { enableMaster = true; master = new NamedList<Object>(); } if (enableMaster) { includeConfFiles = (String) master.get(CONF_FILES); if (includeConfFiles != null && includeConfFiles.trim().length() > 0) { List<String> files = Arrays.asList(includeConfFiles.split(",")); for (String file : files) { if (file.trim().length() == 0) continue; String[] strs = file.trim().split(":"); // if there is an alias add it or it is null confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); } LOG.info("Replication enabled for following config files: " + includeConfFiles); } List backup = master.getAll("backupAfter"); boolean backupOnCommit = backup.contains("commit"); boolean backupOnOptimize = !backupOnCommit && backup.contains("optimize"); List replicateAfter = master.getAll(REPLICATE_AFTER); replicateOnCommit = replicateAfter.contains("commit"); replicateOnOptimize = !replicateOnCommit && replicateAfter.contains("optimize"); if (!replicateOnCommit && !replicateOnOptimize) { replicateOnCommit = true; } // if we only want to replicate on optimize, we need the deletion policy to // save the last optimized commit point. if (replicateOnOptimize) { IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); if (policy instanceof SolrDeletionPolicy) { SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy) policy; if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) { solrPolicy.setMaxOptimizedCommitsToKeep(1); } } else { LOG.warn("Replication can't call setMaxOptimizedCommitsToKeep on " + policy); } } if (replicateOnOptimize || backupOnOptimize) { core.getUpdateHandler() .registerOptimizeCallback(getEventListener(backupOnOptimize, replicateOnOptimize)); } if (replicateOnCommit || backupOnCommit) { replicateOnCommit = true; core.getUpdateHandler().registerCommitCallback(getEventListener(backupOnCommit, replicateOnCommit)); } if (replicateAfter.contains("startup")) { replicateOnStart = true; RefCounted<SolrIndexSearcher> s = core.getNewestSearcher(false); try { DirectoryReader reader = s == null ? null : s.get().getIndexReader(); if (reader != null && reader.getIndexCommit() != null && reader.getIndexCommit().getGeneration() != 1L) { try { if (replicateOnOptimize) { Collection<IndexCommit> commits = DirectoryReader.listCommits(reader.directory()); for (IndexCommit ic : commits) { if (ic.getSegmentCount() == 1) { if (indexCommitPoint == null || indexCommitPoint.getGeneration() < ic.getGeneration()) indexCommitPoint = ic; } } } else { indexCommitPoint = reader.getIndexCommit(); } } finally { // We don't need to save commit points for replication, the SolrDeletionPolicy // always saves the last commit point (and the last optimized commit point, if needed) /*** if(indexCommitPoint != null){ core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration()); } ***/ } } // ensure the writer is init'd so that we have a list of commit points RefCounted<IndexWriter> iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); iw.decref(); } catch (IOException e) { LOG.warn("Unable to get IndexCommit on startup", e); } finally { if (s != null) s.decref(); } } String reserve = (String) master.get(RESERVE); if (reserve != null && !reserve.trim().equals("")) { reserveCommitDuration = SnapPuller.readInterval(reserve); } LOG.info("Commits will be reserved for " + reserveCommitDuration); isMaster = true; } }
From source file:org.elasticsearch.index.engine.EngineDiskUtils.java
License:Apache License
/** * Creates a new empty translog and associates it with an existing lucene index. *///w ww . j a va 2 s . co m public static void createNewTranslog(final Directory dir, final Path translogPath, long initialGlobalCheckpoint, final ShardId shardId) throws IOException { if (Assertions.ENABLED) { final List<IndexCommit> existingCommits = DirectoryReader.listCommits(dir); assert existingCommits.size() == 1 : "creating a translog translog should have one commit, commits[" + existingCommits + "]"; SequenceNumbers.CommitInfo commitInfo = Store.loadSeqNoInfo(existingCommits.get(0)); assert commitInfo.localCheckpoint >= initialGlobalCheckpoint : "trying to create a shard whose local checkpoint [" + commitInfo.localCheckpoint + "] is < global checkpoint [" + initialGlobalCheckpoint + "]"; } try (IndexWriter writer = newIndexWriter(false, dir)) { final String translogUuid = Translog.createEmptyTranslog(translogPath, initialGlobalCheckpoint, shardId); final Map<String, String> map = new HashMap<>(); map.put(Translog.TRANSLOG_GENERATION_KEY, "1"); map.put(Translog.TRANSLOG_UUID_KEY, translogUuid); updateCommitData(writer, map); } }
From source file:org.elasticsearch.index.translog.TestTranslog.java
License:Apache License
/** * Lists all existing commits in a given index path, then read the minimum translog generation that will be used in recoverFromTranslog. */// w w w . j av a 2s .co m private static long minTranslogGenUsedInRecovery(Path translogPath) throws IOException { try (NIOFSDirectory directory = new NIOFSDirectory(translogPath.getParent().resolve("index"))) { List<IndexCommit> commits = DirectoryReader.listCommits(directory); final String translogUUID = commits.get(commits.size() - 1).getUserData() .get(Translog.TRANSLOG_UUID_KEY); long globalCheckpoint = Translog.readGlobalCheckpoint(translogPath, translogUUID); IndexCommit recoveringCommit = CombinedDeletionPolicy.findSafeCommitPoint(commits, globalCheckpoint); return Long.parseLong(recoveringCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); } }
From source file:org.elasticsearch.index.translog.TruncateTranslogCommand.java
License:Apache License
@Override protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception { boolean batch = options.has(batchMode); Path translogPath = getTranslogPath(options); Path idxLocation = translogPath.getParent().resolve("index"); if (Files.exists(translogPath) == false || Files.isDirectory(translogPath) == false) { throw new ElasticsearchException( "translog directory [" + translogPath + "], must exist and be a directory"); }/*from w w w. jav a 2s . c o m*/ if (Files.exists(idxLocation) == false || Files.isDirectory(idxLocation) == false) { throw new ElasticsearchException( "unable to find a shard at [" + idxLocation + "], which must exist and be a directory"); } // Hold the lock open for the duration of the tool running try (Directory dir = FSDirectory.open(idxLocation, NativeFSLockFactory.INSTANCE); Lock writeLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { Set<Path> translogFiles; try { terminal.println("Checking existing translog files"); translogFiles = filesInDirectory(translogPath); } catch (IOException e) { terminal.println("encountered IOException while listing directory, aborting..."); throw new ElasticsearchException("failed to find existing translog files", e); } // Warn about ES being stopped and files being deleted warnAboutDeletingFiles(terminal, translogFiles, batch); List<IndexCommit> commits; try { terminal.println( "Reading translog UUID information from Lucene commit from shard at [" + idxLocation + "]"); commits = DirectoryReader.listCommits(dir); } catch (IndexNotFoundException infe) { throw new ElasticsearchException("unable to find a valid shard at [" + idxLocation + "]", infe); } // Retrieve the generation and UUID from the existing data Map<String, String> commitData = commits.get(commits.size() - 1).getUserData(); String translogGeneration = commitData.get(Translog.TRANSLOG_GENERATION_KEY); String translogUUID = commitData.get(Translog.TRANSLOG_UUID_KEY); if (translogGeneration == null || translogUUID == null) { throw new ElasticsearchException( "shard must have a valid translog generation and UUID but got: [{}] and: [{}]", translogGeneration, translogUUID); } terminal.println("Translog Generation: " + translogGeneration); terminal.println("Translog UUID : " + translogUUID); Path tempEmptyCheckpoint = translogPath.resolve("temp-" + Translog.CHECKPOINT_FILE_NAME); Path realEmptyCheckpoint = translogPath.resolve(Translog.CHECKPOINT_FILE_NAME); Path tempEmptyTranslog = translogPath.resolve( "temp-" + Translog.TRANSLOG_FILE_PREFIX + translogGeneration + Translog.TRANSLOG_FILE_SUFFIX); Path realEmptyTranslog = translogPath .resolve(Translog.TRANSLOG_FILE_PREFIX + translogGeneration + Translog.TRANSLOG_FILE_SUFFIX); // Write empty checkpoint and translog to empty files long gen = Long.parseLong(translogGeneration); int translogLen = writeEmptyTranslog(tempEmptyTranslog, translogUUID); writeEmptyCheckpoint(tempEmptyCheckpoint, translogLen, gen); terminal.println("Removing existing translog files"); IOUtils.rm(translogFiles.toArray(new Path[] {})); terminal.println("Creating new empty checkpoint at [" + realEmptyCheckpoint + "]"); Files.move(tempEmptyCheckpoint, realEmptyCheckpoint, StandardCopyOption.ATOMIC_MOVE); terminal.println("Creating new empty translog at [" + realEmptyTranslog + "]"); Files.move(tempEmptyTranslog, realEmptyTranslog, StandardCopyOption.ATOMIC_MOVE); // Fsync the translog directory after rename IOUtils.fsync(translogPath, true); } catch (LockObtainFailedException lofe) { throw new ElasticsearchException( "Failed to lock shard's directory at [" + idxLocation + "], is Elasticsearch still running?"); } terminal.println("Done."); }