Example usage for org.apache.lucene.index SnapshotDeletionPolicy snapshot

Introduction

In this page you can find the example usage for org.apache.lucene.index SnapshotDeletionPolicy snapshot.

Prototype

public synchronized IndexCommit snapshot() throws IOException 

Source Link

Document

Snapshots the last commit and returns it.

Usage

From source file:com.leavesfly.lia.admin.Fragments.java

License:Apache License

public void test() throws Exception {
    Directory dir = null;/*from   w  w  w.jav  a  2s  .c om*/
    Analyzer analyzer = null;
    // START
    IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
    SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy);
    IndexWriter writer = new IndexWriter(dir, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
    // END

    try {
        IndexCommit commit = (IndexCommit) snapshotter.snapshot();
        Collection<String> fileNames = commit.getFileNames();
        /*<iterate over & copy files from fileNames>*/
    } finally {
        snapshotter.release();
    }
}

---

From source file:com.mathworks.xzheng.admin.Fragments.java

License:Apache License

public void test() throws Exception {
    Directory dir = null;//from   w  ww.j av  a 2s.  c  om
    Analyzer analyzer = null;
    // START
    IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
    SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy);

    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, analyzer);
    config.setIndexDeletionPolicy(snapshotter);
    IndexWriter writer = new IndexWriter(dir, config);
    // END

    IndexCommit commit = null;
    try {
        commit = (IndexCommit) snapshotter.snapshot();
        Collection<String> fileNames = commit.getFileNames();
        /*<iterate over & copy files from fileNames>*/
    } finally {
        snapshotter.release(commit);
    }
}

---

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void handleBackup(Operation op, BackupRequest req) throws Throwable {
    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;//ww w. j  a  v a 2 s .c  om
    handleMaintenanceImpl(true);
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;
    }
    try {
        // Create a snapshot so the index files won't be deleted.
        snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(),
                FILE_PATH_LUCENE);

        // Add the files in the commit to a zip file.
        List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames());
        req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc());

        op.setBody(req).complete();
    } catch (Exception e) {
        this.logSevere(e);
        throw e;
    } finally {
        if (snapshotter != null) {
            snapshotter.release(commit);
        }
        w.deleteUnusedFiles();
    }
}

---

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexBackupService.java

License:Open Source License

private void takeSnapshot(Path destinationPath, boolean isZipBackup, InternalDocumentIndexInfo indexInfo)
        throws IOException {

    IndexWriter writer = indexInfo.indexWriter;
    boolean isInMemoryIndex = indexInfo.indexDirectory == null;

    URI storageSandbox = getHost().getStorageSandbox();

    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;//from w ww  .  j  a  v a 2  s  .co m
    long backupStartTime = System.currentTimeMillis();
    try {
        // Create a snapshot so the index files won't be deleted.
        writer.commit();
        snapshotter = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        if (isZipBackup) {
            Path tempDir = null;
            try {
                List<URI> fileList = new ArrayList<>();
                if (isInMemoryIndex) {
                    tempDir = Files.createTempDirectory("lucene-in-memory-backup");
                    copyInMemoryLuceneIndexToDirectory(commit, tempDir);
                    List<URI> files = Files.list(tempDir).map(Path::toUri).collect(toList());
                    fileList.addAll(files);
                } else {

                    Path indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory);
                    List<URI> files = commit.getFileNames().stream().map(indexDirectoryPath::resolve)
                            .map(Path::toUri).collect(toList());
                    fileList.addAll(files);
                }

                // Add files in the commit to a zip file.
                FileUtils.zipFiles(fileList, destinationPath.toFile());
            } finally {
                if (tempDir != null) {
                    FileUtils.deleteFiles(tempDir.toFile());
                }
            }
        } else {
            // incremental backup

            // create destination dir if not exist
            if (!Files.exists(destinationPath)) {
                Files.createDirectory(destinationPath);
            }

            Set<String> sourceFileNames = new HashSet<>(commit.getFileNames());

            Set<String> destFileNames = Files.list(destinationPath).filter(Files::isRegularFile)
                    .map(path -> path.getFileName().toString()).collect(toSet());

            Path tempDir = null;
            try {
                Path indexDirectoryPath;
                if (isInMemoryIndex) {
                    // copy files into temp directory and point index directory path to temp dir
                    tempDir = Files.createTempDirectory("lucene-in-memory-backup");
                    copyInMemoryLuceneIndexToDirectory(commit, tempDir);
                    indexDirectoryPath = tempDir;
                } else {
                    indexDirectoryPath = Paths.get(storageSandbox).resolve(indexInfo.indexDirectory);
                }

                // add files exist in source but not in dest
                Set<String> toAdd = new HashSet<>(sourceFileNames);
                toAdd.removeAll(destFileNames);
                for (String filename : toAdd) {
                    Path source = indexDirectoryPath.resolve(filename);
                    Path target = destinationPath.resolve(filename);
                    Files.copy(source, target);
                }

                // delete files exist in dest but not in source
                Set<String> toDelete = new HashSet<>(destFileNames);
                toDelete.removeAll(sourceFileNames);
                for (String filename : toDelete) {
                    Path path = destinationPath.resolve(filename);
                    Files.delete(path);
                }

                long backupEndTime = System.currentTimeMillis();
                logInfo("Incremental backup performed. dir=%s, added=%d, deleted=%d, took=%dms",
                        destinationPath, toAdd.size(), toDelete.size(), backupEndTime - backupStartTime);
            } finally {
                if (tempDir != null) {
                    FileUtils.deleteFiles(tempDir.toFile());
                }
            }
        }
    } finally {
        if (snapshotter != null && commit != null) {
            snapshotter.release(commit);
        }
        writer.deleteUnusedFiles();
    }
}

---

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void handleBackup(Operation op, BackupRequest req) throws Throwable {
    SnapshotDeletionPolicy snapshotter = null;
    IndexCommit commit = null;/*from   w ww  . j av  a  2s .  c  om*/
    handleMaintenanceImpl(true);
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;
    }
    try {
        // Create a snapshot so the index files won't be deleted.
        snapshotter = (SnapshotDeletionPolicy) w.getConfig().getIndexDeletionPolicy();
        commit = snapshotter.snapshot();

        String indexDirectory = UriUtils.buildUriPath(getHost().getStorageSandbox().getPath(),
                this.indexDirectory);

        // Add the files in the commit to a zip file.
        List<URI> fileList = FileUtils.filesToUris(indexDirectory, commit.getFileNames());
        req.backupFile = FileUtils.zipFiles(fileList, this.indexDirectory + "-" + Utils.getNowMicrosUtc());

        op.setBody(req).complete();
    } catch (Exception e) {
        this.logSevere(e);
        throw e;
    } finally {
        if (snapshotter != null) {
            snapshotter.release(commit);
        }
        w.deleteUnusedFiles();
    }
}

---

From source file:io.datalayer.lucene.snapshot.IndexSnapshotTest.java

License:Apache License

@Test
@Ignore/*from   ww  w.  jav  a2 s .  c o  m*/
public void testSnapshot() throws Exception {

    Directory dir = null;

    IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
    SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(policy);
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44,
            AosAnalyser.NO_LIMIT_TOKEN_COUNT_SIMPLE_ANALYSER);
    conf.setIndexDeletionPolicy(snapshotter);

    IndexWriter writer = new IndexWriter(dir, conf);

    IndexCommit indexCommit = null;
    try {
        indexCommit = snapshotter.snapshot();
        Collection<String> fileNames = indexCommit.getFileNames();
        /* <iterate over & copy files from fileNames> */
    } finally {
        if (indexCommit != null) {
            snapshotter.release(indexCommit);
        }
    }

    writer.close();

}

---

From source file:org.neo4j.index.impl.lucene.legacy.LuceneDataSource.java

License:Open Source License

public ResourceIterator<File> listStoreFiles(boolean includeLogicalLogs) throws IOException { // Never include logical logs since they are of little importance
    final Collection<File> files = new ArrayList<>();
    final Collection<Pair<SnapshotDeletionPolicy, IndexCommit>> snapshots = new ArrayList<>();
    makeSureAllIndexesAreInstantiated();
    for (IndexReference writer : getAllIndexes()) {
        SnapshotDeletionPolicy deletionPolicy = (SnapshotDeletionPolicy) writer.getWriter().getConfig()
                .getIndexDeletionPolicy();
        File indexDirectory = getFileDirectory(baseStorePath, writer.getIdentifier());
        IndexCommit commit;//w ww.ja  v a 2s.c  o  m
        try {
            // Throws IllegalStateException if no commits yet
            commit = deletionPolicy.snapshot();
        } catch (IllegalStateException e) {
            /*
             * This is insane but happens if we try to snapshot an existing index
             * that has no commits. This is a bad API design - it should return null
             * or something. This is not exceptional.
             *
             * For the time being we just do a commit and try again.
             */
            writer.getWriter().commit();
            commit = deletionPolicy.snapshot();
        }

        for (String fileName : commit.getFileNames()) {
            files.add(new File(indexDirectory, fileName));
        }
        snapshots.add(Pair.of(deletionPolicy, commit));
    }
    return new PrefetchingResourceIterator<File>() {
        private final Iterator<File> filesIterator = files.iterator();

        @Override
        protected File fetchNextOrNull() {
            return filesIterator.hasNext() ? filesIterator.next() : null;
        }

        @Override
        public void close() {
            for (Pair<SnapshotDeletionPolicy, IndexCommit> policyAndCommit : snapshots) {
                try {
                    policyAndCommit.first().release(policyAndCommit.other());
                } catch (IOException e) {
                    // TODO What to do?
                    e.printStackTrace();
                }
            }
        }
    };
}

---

From source file:org.neo4j.kernel.api.impl.index.backup.LuceneIndexSnapshotFileIterator.java

License:Open Source License

private LuceneIndexSnapshotFileIterator(File indexDirectory, SnapshotDeletionPolicy snapshotDeletionPolicy)
        throws IOException {
    this.snapshot = snapshotDeletionPolicy.snapshot();
    this.indexDirectory = indexDirectory;
    this.snapshotDeletionPolicy = snapshotDeletionPolicy;
    this.fileNames = snapshot.getFileNames().iterator();
}

---

From source file:org.neo4j.kernel.api.impl.index.backup.WritableIndexSnapshotFileIterator.java

License:Open Source License

WritableIndexSnapshotFileIterator(File indexDirectory, SnapshotDeletionPolicy snapshotDeletionPolicy)
        throws IOException {
    super(indexDirectory, snapshotDeletionPolicy.snapshot());
    this.snapshotDeletionPolicy = snapshotDeletionPolicy;
}

---

From source file:org.ojbc.adapters.analyticaldatastore.personid.IndexedIdentifierGenerationStrategy.java

License:RPL License

/**
 * This method will back up the lucene cache using the specified backup path.
 * It will create a new directory with the format yyyy_MM_dd_HH_mm_ss which will
 * contain the backed up index.  Hot backups are allowed.
 * // w w w.j  a  va 2 s  . c o m
 */
@Override
public String backup() throws Exception {
    SnapshotDeletionPolicy snapshotter = (SnapshotDeletionPolicy) indexWriter.getConfig()
            .getIndexDeletionPolicy();

    IndexCommit commit = null;
    String backupFileDirectory = "";

    try {
        LocalDateTime today = LocalDateTime.now();
        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu_MM_dd_HH_mm_ss");
        String dateTimeStamp = today.format(dtf);

        if (!(indexBackupRoot.endsWith("/") || indexBackupRoot.endsWith("\\"))) {
            indexBackupRoot = indexBackupRoot + System.getProperty("file.separator");
        }

        backupFileDirectory = indexBackupRoot + dateTimeStamp + System.getProperty("file.separator");

        commit = snapshotter.snapshot();
        for (String fileName : commit.getFileNames()) {
            log.debug("File Name: " + fileName);

            File srcFile = new File(indexDirectoryPath + System.getProperty("file.separator") + fileName);
            File destFile = new File(backupFileDirectory + System.getProperty("file.separator") + fileName);

            FileUtils.copyFile(srcFile, destFile);

        }
    } catch (Exception e) {
        log.error("Exception", e);
    } finally {
        snapshotter.release(commit);
    }

    return backupFileDirectory;
}

---