List of usage examples for org.apache.lucene.index IndexCommit getFileNames
public abstract Collection<String> getFileNames() throws IOException;
From source file:com.b2international.index.compat.SingleDirectoryIndexImpl.java
License:Apache License
@Override public List<String> listFiles(final String snapshotId) throws IOException { checkNotDisposed();//from w w w .j a v a 2 s. c om checkNotNull(snapshotId, "Snapshot identifier may not be null."); final Set<String> result = Sets.newHashSet(); final IndexCommit indexCommit = heldSnapshots.get(snapshotId); if (null == indexCommit) { return Lists.newArrayList(); } final File basePath = getIndexBasePath(); if (null == basePath) { return Lists.newArrayList(); } final Path base = Paths.get(basePath.getAbsolutePath()); final Path actual = Paths.get(indexDirectory.getAbsolutePath()); final Path relativePath = actual.relativize(base); final Collection<String> fileNames = indexCommit.getFileNames(); for (final String fileName : fileNames) { final File indexFilePath = new File(indexDirectory, fileName); // Only collect files from this folder if (indexFilePath.exists() && indexFilePath.isFile()) { result.add(relativePath.resolve(fileName).toString()); } } return Ordering.natural().sortedCopy(result); }
From source file:com.leavesfly.lia.admin.Fragments.java
License:Apache License
public void test() throws Exception { Directory dir = null;//from ww w. j a v a 2 s.c o m Analyzer analyzer = null; // START IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy); IndexWriter writer = new IndexWriter(dir, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); // END try { IndexCommit commit = (IndexCommit) snapshotter.snapshot(); Collection<String> fileNames = commit.getFileNames(); /*<iterate over & copy files from fileNames>*/ } finally { snapshotter.release(); } }
From source file:com.liferay.portal.search.lucene.dump.IndexCommitMetaInfo.java
License:Open Source License
public IndexCommitMetaInfo(IndexCommit indexCommit) throws IOException { if (indexCommit == null) { _empty = true;/*from www . ja v a 2s. com*/ return; } List<String> fileNames = new ArrayList<String>(indexCommit.getFileNames()); _segments = new ArrayList<Segment>(fileNames.size()); Directory directory = indexCommit.getDirectory(); for (String fileName : fileNames) { Segment segment = new Segment(fileName, directory.fileLength(fileName)); _segments.add(segment); } _generation = indexCommit.getGeneration(); }
From source file:com.mathworks.xzheng.admin.Fragments.java
License:Apache License
public void test() throws Exception { Directory dir = null;//w w w.j av a 2s.c om Analyzer analyzer = null; // START IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, analyzer); config.setIndexDeletionPolicy(snapshotter); IndexWriter writer = new IndexWriter(dir, config); // END IndexCommit commit = null; try { commit = (IndexCommit) snapshotter.snapshot(); Collection<String> fileNames = commit.getFileNames(); /*<iterate over & copy files from fileNames>*/ } finally { snapshotter.release(commit); } }
From source file:com.nearinfinity.mele.zookeeper.ZookeeperIndexDeletionPolicy.java
License:Apache License
@Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { List<String> filesCurrentlyBeingReferenced = getListOfReferencedFiles(zk, indexRefPath); int size = commits.size(); Collection<String> previouslyReferencedFiles = new TreeSet<String>(); OUTER: for (int i = size - 2; i >= 0; i--) { IndexCommit indexCommit = commits.get(i); LOG.info("Processing index commit generation " + indexCommit.getGeneration()); Collection<String> fileNames = new TreeSet<String>(indexCommit.getFileNames()); // remove all filenames that were references in newer index commits, // this way older index commits can be released without the fear of // broken references. fileNames.removeAll(previouslyReferencedFiles); for (String fileName : fileNames) { if (filesCurrentlyBeingReferenced.contains(fileName)) { previouslyReferencedFiles.addAll(fileNames); continue OUTER; }/*from w w w . j a v a2s. co m*/ } LOG.info("Index Commit " + indexCommit.getGeneration() + " no longer needed, releasing " + fileNames); indexCommit.delete(); } }
From source file:com.qwazr.search.index.IndexInstance.java
License:Apache License
final synchronized BackupStatus backup(Integer keepLastCount) throws IOException, InterruptedException { Semaphore sem = schema.acquireReadSemaphore(); try {/*from w ww.j a va2 s .com*/ File backupdir = null; final IndexCommit commit = snapshotDeletionPolicy.snapshot(); try { int files_count = 0; long bytes_size = 0; if (!fileSet.backupDirectory.exists()) fileSet.backupDirectory.mkdir(); backupdir = new File(fileSet.backupDirectory, Long.toString(commit.getGeneration())); if (!backupdir.exists()) backupdir.mkdir(); if (!backupdir.exists()) throw new IOException("Cannot create the backup directory: " + backupdir); for (String fileName : commit.getFileNames()) { File sourceFile = new File(fileSet.dataDirectory, fileName); File targetFile = new File(backupdir, fileName); files_count++; bytes_size += sourceFile.length(); if (targetFile.exists() && targetFile.length() == sourceFile.length() && targetFile.lastModified() == sourceFile.lastModified()) continue; FileUtils.copyFile(sourceFile, targetFile, true); } purgeBackups(keepLastCount); return new BackupStatus(commit.getGeneration(), backupdir.lastModified(), bytes_size, files_count); } catch (IOException e) { if (backupdir != null) FileUtils.deleteQuietly(backupdir); throw e; } finally { snapshotDeletionPolicy.release(commit); } } finally { if (sem != null) sem.release(); } }
From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java
License:Open Source License
private void handleBackup(Operation op, BackupRequest req) throws Throwable { SnapshotDeletionPolicy snapshotter = null; IndexCommit commit = null; handleMaintenanceImpl(true);//w w w . j a v a2s .c o m IndexWriter w = this.writer; if (w == null) { op.fail(new CancellationException()); return; } try { // Create a snapshot so the index files won't be deleted. snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy(); commit = snapshotter.snapshot(); String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(), FILE_PATH_LUCENE); // Add the files in the commit to a zip file. List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames()); req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc()); op.setBody(req).complete(); } catch (Exception e) { this.logSevere(e); throw e; } finally { if (snapshotter != null) { snapshotter.release(commit); } w.deleteUnusedFiles(); } }
From source file:com.vmware.xenon.services.common.LuceneDocumentIndexBackupService.java
License:Open Source License
private void takeSnapshot(Path destinationPath, boolean isZipBackup, InternalDocumentIndexInfo indexInfo) throws IOException { IndexWriter writer = indexInfo.indexWriter; boolean isInMemoryIndex = indexInfo.indexDirectory == null; URI storageSandbox = getHost().getStorageSandbox(); SnapshotDeletionPolicy snapshotter = null; IndexCommit commit = null; long backupStartTime = System.currentTimeMillis(); try {// w ww .ja v a 2 s . c o m // Create a snapshot so the index files won't be deleted. writer.commit(); snapshotter = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy(); commit = snapshotter.snapshot(); if (isZipBackup) { Path tempDir = null; try { List<URI> fileList = new ArrayList<>(); if (isInMemoryIndex) { tempDir = Files.createTempDirectory("lucene-in-memory-backup"); copyInMemoryLuceneIndexToDirectory(commit, tempDir); List<URI> files = Files.list(tempDir).map(Path::toUri).collect(toList()); fileList.addAll(files); } else { Path indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory); List<URI> files = commit.getFileNames().stream().map(indexDirectoryPath::resolve) .map(Path::toUri).collect(toList()); fileList.addAll(files); } // Add files in the commit to a zip file. FileUtils.zipFiles(fileList, destinationPath.toFile()); } finally { if (tempDir != null) { FileUtils.deleteFiles(tempDir.toFile()); } } } else { // incremental backup // create destination dir if not exist if (!Files.exists(destinationPath)) { Files.createDirectory(destinationPath); } Set<String> sourceFileNames = new HashSet<>(commit.getFileNames()); Set<String> destFileNames = Files.list(destinationPath).filter(Files::isRegularFile) .map(path -> path.getFileName().toString()).collect(toSet()); Path tempDir = null; try { Path indexDirectoryPath; if (isInMemoryIndex) { // copy files into temp directory and point index directory path to temp dir tempDir = Files.createTempDirectory("lucene-in-memory-backup"); copyInMemoryLuceneIndexToDirectory(commit, tempDir); indexDirectoryPath = tempDir; } else { indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory); } // add files exist in source but not in dest Set<String> toAdd = new HashSet<>(sourceFileNames); toAdd.removeAll(destFileNames); for (String filename : toAdd) { Path source = indexDirectoryPath.resolve(filename); Path target = destinationPath.resolve(filename); Files.copy(source, target); } // delete files exist in dest but not in source Set<String> toDelete = new HashSet<>(destFileNames); toDelete.removeAll(sourceFileNames); for (String filename : toDelete) { Path path = destinationPath.resolve(filename); Files.delete(path); } long backupEndTime = System.currentTimeMillis(); logInfo("Incremental backup performed. dir=%s, added=%d, deleted=%d, took=%dms", destinationPath, toAdd.size(), toDelete.size(), backupEndTime - backupStartTime); } finally { if (tempDir != null) { FileUtils.deleteFiles(tempDir.toFile()); } } } } finally { if (snapshotter != null && commit != null) { snapshotter.release(commit); } writer.deleteUnusedFiles(); } }
From source file:com.vmware.xenon.services.common.LuceneDocumentIndexBackupService.java
License:Open Source License
private void copyInMemoryLuceneIndexToDirectory(IndexCommit commit, Path directoryPath) throws IOException { Directory from = commit.getDirectory(); try (Directory to = new NIOFSDirectory(directoryPath)) { for (String filename : commit.getFileNames()) { to.copyFrom(from, filename, filename, IOContext.DEFAULT); }//from ww w . j a v a 2 s . c o m } }
From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java
License:Open Source License
private void handleBackup(Operation op, BackupRequest req) throws Throwable { SnapshotDeletionPolicy snapshotter = null; IndexCommit commit = null; handleMaintenanceImpl(true);// w w w .j a v a2 s . c o m IndexWriter w = this.writer; if (w == null) { op.fail(new CancellationException()); return; } try { // Create a snapshot so the index files won't be deleted. snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy(); commit = snapshotter.snapshot(); String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(), this.indexDirectory); // Add the files in the commit to a zip file. List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames()); req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc()); op.setBody(req).complete(); } catch (Exception e) { this.logSevere(e); throw e; } finally { if (snapshotter != null) { snapshotter.release(commit); } w.deleteUnusedFiles(); } }