Example usage for org.apache.lucene.index SnapshotDeletionPolicy release

Introduction

In this page you can find the example usage for org.apache.lucene.index SnapshotDeletionPolicy release.

Prototype

public synchronized void release(IndexCommit commit) throws IOException 

Source Link

Document

Release a snapshotted commit.

Usage

From source file:com.mathworks.xzheng.admin.Fragments.java

License:Apache License

public void test() throws Exception {
    Directory dir = null;// w  w  w  .j  a  va  2 s . co  m
    Analyzer analyzer = null;
    // START
    IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
    SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy);

    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, analyzer);
    config.setIndexDeletionPolicy(snapshotter);
    IndexWriter writer = new IndexWriter(dir, config);
    // END

    IndexCommit commit = null;
    try {
        commit = (IndexCommit) snapshotter.snapshot();
        Collection<String> fileNames = commit.getFileNames();
        /*<iterate over & copy files from fileNames>*/
    } finally {
        snapshotter.release(commit);
    }
}

---

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void handleBackup(Operation op, BackupRequest req) throws Throwable {
    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;/*from  w  w  w .j  a v  a2  s .co  m*/
    handleMaintenanceImpl(true);
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;
    }
    try {
        // Create a snapshot so the index files won't be deleted.
        snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(),
                FILE_PATH_LUCENE);

        // Add the files in the commit to a zip file.
        List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames());
        req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc());

        op.setBody(req).complete();
    } catch (Exception e) {
        this.logSevere(e);
        throw e;
    } finally {
        if (snapshotter != null) {
            snapshotter.release(commit);
        }
        w.deleteUnusedFiles();
    }
}

---

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexBackupService.java

License:Open Source License

private void takeSnapshot(Path destinationPath, boolean isZipBackup, InternalDocumentIndexInfo indexInfo)
        throws IOException {

    IndexWriter writer = indexInfo.indexWriter;
    boolean isInMemoryIndex = indexInfo.indexDirectory == null;

    URI storageSandbox = getHost().getStorageSandbox();

    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;/* w w w  .j  av a 2  s.  c o  m*/
    long backupStartTime = System.currentTimeMillis();
    try {
        // Create a snapshot so the index files won't be deleted.
        writer.commit();
        snapshotter = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        if (isZipBackup) {
            Path tempDir = null;
            try {
                List<URI> fileList = new ArrayList<>();
                if (isInMemoryIndex) {
                    tempDir = Files.createTempDirectory("lucene-in-memory-backup");
                    copyInMemoryLuceneIndexToDirectory(commit, tempDir);
                    List<URI> files = Files.list(tempDir).map(Path::toUri).collect(toList());
                    fileList.addAll(files);
                } else {

                    Path indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory);
                    List<URI> files = commit.getFileNames().stream().map(indexDirectoryPath::resolve)
                            .map(Path::toUri).collect(toList());
                    fileList.addAll(files);
                }

                // Add files in the commit to a zip file.
                FileUtils.zipFiles(fileList, destinationPath.toFile());
            } finally {
                if (tempDir != null) {
                    FileUtils.deleteFiles(tempDir.toFile());
                }
            }
        } else {
            // incremental backup

            // create destination dir if not exist
            if (!Files.exists(destinationPath)) {
                Files.createDirectory(destinationPath);
            }

            Set<String> sourceFileNames = new HashSet<>(commit.getFileNames());

            Set<String> destFileNames = Files.list(destinationPath).filter(Files::isRegularFile)
                    .map(path -> path.getFileName().toString()).collect(toSet());

            Path tempDir = null;
            try {
                Path indexDirectoryPath;
                if (isInMemoryIndex) {
                    // copy files into temp directory and point index directory path to temp dir
                    tempDir = Files.createTempDirectory("lucene-in-memory-backup");
                    copyInMemoryLuceneIndexToDirectory(commit, tempDir);
                    indexDirectoryPath = tempDir;
                } else {
                    indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory);
                }

                // add files exist in source but not in dest
                Set<String> toAdd = new HashSet<>(sourceFileNames);
                toAdd.removeAll(destFileNames);
                for (String filename : toAdd) {
                    Path source = indexDirectoryPath.resolve(filename);
                    Path target = destinationPath.resolve(filename);
                    Files.copy(source, target);
                }

                // delete files exist in dest but not in source
                Set<String> toDelete = new HashSet<>(destFileNames);
                toDelete.removeAll(sourceFileNames);
                for (String filename : toDelete) {
                    Path path = destinationPath.resolve(filename);
                    Files.delete(path);
                }

                long backupEndTime = System.currentTimeMillis();
                logInfo("Incremental backup performed. dir=%s, added=%d, deleted=%d, took=%dms",
                        destinationPath, toAdd.size(), toDelete.size(), backupEndTime - backupStartTime);
            } finally {
                if (tempDir != null) {
                    FileUtils.deleteFiles(tempDir.toFile());
                }
            }
        }
    } finally {
        if (snapshotter != null && commit != null) {
            snapshotter.release(commit);
        }
        writer.deleteUnusedFiles();
    }
}

---

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void handleBackup(Operation op, BackupRequest req) throws Throwable {
    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;// ww  w.j  a  va 2s  . c o  m
    handleMaintenanceImpl(true);
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;
    }
    try {
        // Create a snapshot so the index files won't be deleted.
        snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(),
                this.indexDirectory);

        // Add the files in the commit to a zip file.
        List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames());
        req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc());

        op.setBody(req).complete();
    } catch (Exception e) {
        this.logSevere(e);
        throw e;
    } finally {
        if (snapshotter != null) {
            snapshotter.release(commit);
        }
        w.deleteUnusedFiles();
    }
}

---

From source file:io.datalayer.lucene.snapshot.IndexSnapshotTest.java

License:Apache License

@Test
@Ignore//from ww w .j  a v  a2  s  . c  o  m
public void testSnapshot() throws Exception {

    Directory dir = null;

    IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
    SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy);
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44,
            AosAnalyser.NO_LIMIT_TOKEN_COUNT_SIMPLE_ANALYSER);
    conf.setIndexDeletionPolicy(snapshotter);

    IndexWriter writer = new IndexWriter(dir, conf);

    IndexCommit indexCommit = null;
    try {
        indexCommit = snapshotter.snapshot();
        Collection<String> fileNames = indexCommit.getFileNames();
        /* <iterate over & copy files from fileNames> */
    } finally {
        if (indexCommit != null) {
            snapshotter.release(indexCommit);
        }
    }

    writer.close();

}

---

From source file:org.neo4j.index.impl.lucene.LuceneDataSource.java

License:Open Source License

public ResourceIterator<File> listStoreFiles(boolean includeLogicalLogs) throws IOException { // Never include logical logs since they are of little importance
    final Collection<File> files = new ArrayList<>();
    final Collection<SnapshotDeletionPolicy> snapshots = new ArrayList<>();
    makeSureAllIndexesAreInstantiated();
    for (IndexReference writer : getAllIndexes()) {
        SnapshotDeletionPolicy deletionPolicy = (SnapshotDeletionPolicy) writer.getWriter().getConfig()
                .getIndexDeletionPolicy();
        File indexDirectory = getFileDirectory(baseStorePath, writer.getIdentifier());
        try {//from  w w w  .  ja v  a2  s  .c om
            // Throws IllegalStateException if no commits yet
            IndexCommit commit = deletionPolicy.snapshot(SNAPSHOT_ID);
            for (String fileName : commit.getFileNames()) {
                files.add(new File(indexDirectory, fileName));
            }
            snapshots.add(deletionPolicy);
        } catch (IllegalStateException e) {
            // TODO Review this
            /*
             * This is insane but happens if we try to snapshot an existing index
             * that has no commits. This is a bad API design - it should return null
             * or something. This is not exceptional.
             */
        }
    }
    return new PrefetchingResourceIterator<File>() {
        private final Iterator<File> filesIterator = files.iterator();

        @Override
        protected File fetchNextOrNull() {
            return filesIterator.hasNext() ? filesIterator.next() : null;
        }

        @Override
        public void close() {
            for (SnapshotDeletionPolicy deletionPolicy : snapshots) {
                try {
                    deletionPolicy.release(SNAPSHOT_ID);
                } catch (IOException e) {
                    // TODO What to do?
                    e.printStackTrace();
                }
            }
        }
    };
}

---

From source file:org.ojbc.adapters.analyticaldatastore.personid.IndexedIdentifierGenerationStrategy.java

License:RPL License

/**
 * This method will back up the lucene cache using the specified backup path.
 * It will create a new directory with the format yyyy_MM_dd_HH_mm_ss which will
 * contain the backed up index.  Hot backups are allowed.
 * //from ww  w .  j a va 2s  .co m
 */
@Override
public String backup() throws Exception {
    SnapshotDeletionPolicy snapshotter = (SnapshotDeletionPolicy) indexWriter.getConfig()
            .getIndexDeletionPolicy();

    IndexCommit commit = null;
    String backupFileDirectory = "";

    try {
        LocalDateTime today = LocalDateTime.now();
        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu_MM_dd_HH_mm_ss");
        String dateTimeStamp = today.format(dtf);

        if (!(indexBackupRoot.endsWith("/") || indexBackupRoot.endsWith("\\"))) {
            indexBackupRoot = indexBackupRoot + System.getProperty("file.separator");
        }

        backupFileDirectory = indexBackupRoot + dateTimeStamp + System.getProperty("file.separator");

        commit = snapshotter.snapshot();
        for (String fileName : commit.getFileNames()) {
            log.debug("File Name: " + fileName);

            File srcFile = new File(indexDirectoryPath + System.getProperty("file.separator") + fileName);
            File destFile = new File(backupFileDirectory + System.getProperty("file.separator") + fileName);

            FileUtils.copyFile(srcFile, destFile);

        }
    } catch (Exception e) {
        log.error("Exception", e);
    } finally {
        snapshotter.release(commit);
    }

    return backupFileDirectory;
}

---