List of usage examples for org.apache.lucene.index IndexCommit getGeneration
public abstract long getGeneration();
From source file:com.liferay.portal.search.lucene.dump.IndexCommitMetaInfo.java
License:Open Source License
public IndexCommitMetaInfo(IndexCommit indexCommit) throws IOException { if (indexCommit == null) { _empty = true;/*w ww. j ava2 s. c om*/ return; } List<String> fileNames = new ArrayList<String>(indexCommit.getFileNames()); _segments = new ArrayList<Segment>(fileNames.size()); Directory directory = indexCommit.getDirectory(); for (String fileName : fileNames) { Segment segment = new Segment(fileName, directory.fileLength(fileName)); _segments.add(segment); } _generation = indexCommit.getGeneration(); }
From source file:com.nearinfinity.mele.zookeeper.ZookeeperIndexDeletionPolicy.java
License:Apache License
@Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { List<String> filesCurrentlyBeingReferenced = getListOfReferencedFiles(zk, indexRefPath); int size = commits.size(); Collection<String> previouslyReferencedFiles = new TreeSet<String>(); OUTER: for (int i = size - 2; i >= 0; i--) { IndexCommit indexCommit = commits.get(i); LOG.info("Processing index commit generation " + indexCommit.getGeneration()); Collection<String> fileNames = new TreeSet<String>(indexCommit.getFileNames()); // remove all filenames that were references in newer index commits, // this way older index commits can be released without the fear of // broken references. fileNames.removeAll(previouslyReferencedFiles); for (String fileName : fileNames) { if (filesCurrentlyBeingReferenced.contains(fileName)) { previouslyReferencedFiles.addAll(fileNames); continue OUTER; }//from w w w. j av a 2 s . c o m } LOG.info("Index Commit " + indexCommit.getGeneration() + " no longer needed, releasing " + fileNames); indexCommit.delete(); } }
From source file:com.qwazr.search.index.IndexInstance.java
License:Apache License
final synchronized BackupStatus backup(Integer keepLastCount) throws IOException, InterruptedException { Semaphore sem = schema.acquireReadSemaphore(); try {// w ww. j a v a2 s . c om File backupdir = null; final IndexCommit commit = snapshotDeletionPolicy.snapshot(); try { int files_count = 0; long bytes_size = 0; if (!fileSet.backupDirectory.exists()) fileSet.backupDirectory.mkdir(); backupdir = new File(fileSet.backupDirectory, Long.toString(commit.getGeneration())); if (!backupdir.exists()) backupdir.mkdir(); if (!backupdir.exists()) throw new IOException("Cannot create the backup directory: " + backupdir); for (String fileName : commit.getFileNames()) { File sourceFile = new File(fileSet.dataDirectory, fileName); File targetFile = new File(backupdir, fileName); files_count++; bytes_size += sourceFile.length(); if (targetFile.exists() && targetFile.length() == sourceFile.length() && targetFile.lastModified() == sourceFile.lastModified()) continue; FileUtils.copyFile(sourceFile, targetFile, true); } purgeBackups(keepLastCount); return new BackupStatus(commit.getGeneration(), backupdir.lastModified(), bytes_size, files_count); } catch (IOException e) { if (backupdir != null) FileUtils.deleteQuietly(backupdir); throw e; } finally { snapshotDeletionPolicy.release(commit); } } finally { if (sem != null) sem.release(); } }
From source file:org.apache.blur.index.IndexDeletionPolicyReader.java
License:Apache License
private boolean isStillInUse(IndexCommit commit) { long generation = commit.getGeneration(); return _gens.contains(generation); }
From source file:org.apache.blur.manager.writer.SnapshotIndexDeletionPolicy.java
License:Apache License
@Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { int size = commits.size(); for (int i = 0; i < size - 1; i++) { IndexCommit indexCommit = commits.get(i); long generation = indexCommit.getGeneration(); if (!_generationsToNames.containsKey(generation)) { indexCommit.delete();//from w ww.j a va 2s.com } } }
From source file:org.apache.blur.manager.writer.SnapshotIndexDeletionPolicy.java
License:Apache License
public void createSnapshot(String name, DirectoryReader reader, String context) throws IOException { if (_namesToGenerations.containsKey(name)) { throw new IOException("Snapshot [" + name + "] already exists."); }/*from ww w . j av a 2s .c o m*/ LOG.info("Creating snapshot [{0}] in [{1}].", name, context); IndexCommit indexCommit = reader.getIndexCommit(); long generation = indexCommit.getGeneration(); _namesToGenerations.put(name, generation); Set<String> names = _generationsToNames.get(generation); if (names == null) { names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); _generationsToNames.put(generation, names); } names.add(name); storeGenerations(); }
From source file:org.apache.blur.mapreduce.lib.BlurInputFormat.java
License:Apache License
private static IndexCommit findIndexCommit(List<IndexCommit> listCommits, long generation, Path shardDir) throws IOException { for (IndexCommit commit : listCommits) { if (commit.getGeneration() == generation) { return commit; }// w w w . j ava2 s . c om } throw new IOException("Generation [" + generation + "] not found in shard [" + shardDir + "]"); }
From source file:org.apache.solr.core.snapshots.SolrSnapshotManager.java
License:Apache License
/** * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number. * * @param core The Solr core/*from w ww . j a va2 s. com*/ * @param dir The index directory storing the snapshot. * @param gen The generation number of the {@linkplain IndexCommit} to be deleted. * @throws IOException in case of I/O errors. */ public static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, final long gen) throws IOException { deleteSnapshotIndexFiles(core, dir, new IndexDeletionPolicy() { @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { for (IndexCommit ic : commits) { if (gen == ic.getGeneration()) { log.info("Deleting non-snapshotted index commit with generation {}", ic.getGeneration()); ic.delete(); } } } @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { } }); }
From source file:org.apache.solr.core.snapshots.SolrSnapshotManager.java
License:Apache License
/** * This method deletes index files not associated with the specified <code>snapshots</code>. * * @param core The Solr core//from w w w . ja v a 2s .c o m * @param dir The index directory storing the snapshot. * @param snapshots The snapshots to be preserved. * @throws IOException in case of I/O errors. */ public static void deleteNonSnapshotIndexFiles(SolrCore core, Directory dir, Collection<SnapshotMetaData> snapshots) throws IOException { final Set<Long> genNumbers = new HashSet<>(); for (SnapshotMetaData m : snapshots) { genNumbers.add(m.getGenerationNumber()); } deleteSnapshotIndexFiles(core, dir, new IndexDeletionPolicy() { @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { for (IndexCommit ic : commits) { if (!genNumbers.contains(ic.getGeneration())) { log.info("Deleting non-snapshotted index commit with generation {}", ic.getGeneration()); ic.delete(); } } } @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { } }); }
From source file:org.apache.solr.core.SolrDeletionPolicy.java
License:Apache License
private void updateCommits(List<? extends IndexCommit> commits) { // to be safe, we should only call delete on a commit point passed to us // in this specific call (may be across diff IndexWriter instances). // this will happen rarely, so just synchronize everything // for safety and to avoid race conditions synchronized (this) { long maxCommitAgeTimeStamp = -1L; IndexCommit newest = commits.get(commits.size() - 1); // SOLR-4547: Removed the filenames from this log entry because this // method is only called from methods that have just logged them // at DEBUG. log.info("newest commit generation = " + newest.getGeneration()); int singleSegKept = (newest.getSegmentCount() == 1) ? 1 : 0; int totalKept = 1; // work our way from newest to oldest, skipping the first since we always want to keep it. for (int i = commits.size() - 2; i >= 0; i--) { IndexCommit commit = commits.get(i); // delete anything too old, regardless of other policies try { if (maxCommitAge != null) { if (maxCommitAgeTimeStamp == -1) { DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.ROOT); maxCommitAgeTimeStamp = dmp.parseMath(maxCommitAge).getTime(); }/*from ww w.j av a2 s. c o m*/ if (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) < maxCommitAgeTimeStamp) { commit.delete(); continue; } } } catch (Exception e) { log.warn("Exception while checking commit point's age for deletion", e); } if (singleSegKept < maxOptimizedCommitsToKeep && commit.getSegmentCount() == 1) { totalKept++; singleSegKept++; continue; } if (totalKept < maxCommitsToKeep) { totalKept++; continue; } commit.delete(); } } // end synchronized }