cn.hbu.cs.esearch.index.DiskLuceneIndexDataLoader.java Source code

Introduction

Here is the source code for cn.hbu.cs.esearch.index.DiskLuceneIndexDataLoader.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cn.hbu.cs.esearch.index;

import java.io.IOException;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.Collection;
import java.util.Comparator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.similarities.Similarity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cn.hbu.cs.esearch.core.EsearchException;
import cn.hbu.cs.esearch.core.EsearchHealth;
import cn.hbu.cs.esearch.document.EsearchIndexable;
import it.unimi.dsi.fastutil.longs.LongSet;

public class DiskLuceneIndexDataLoader<R extends IndexReader> extends LuceneIndexDataLoader<R> {

    public static final Logger log = LoggerFactory.getLogger(DiskLuceneIndexDataLoader.class);
    private final Object optimizeMonitor;
    private volatile OptimizeScheduler optScheduler;

    public DiskLuceneIndexDataLoader(Analyzer analyzer, Similarity similarity, SearchIndexManager<R> idxMgr,
            Comparator<String> comparator) {
        super(analyzer, similarity, idxMgr, comparator);
        optimizeMonitor = new Object();
    }

    public void setOptimizeScheduler(OptimizeScheduler scheduler) {
        optScheduler = scheduler;
    }

    public OptimizeScheduler getOptimizeScheduler() {
        return optScheduler;
    }

    @Override
    protected BaseSearchIndex<R> getSearchIndex() {
        return _idxMgr.getDiskIndex();
    }

    @Override
    protected void propagateDeletes(LongSet delDocs) throws IOException {
        // do nothing
    }

    @Override
    protected void commitPropagatedDeletes() throws IOException {
        // do nothing
    }

    @Override
    public void consume(Collection<DataEvent<EsearchIndexable>> events) throws EsearchException {
        // updates the in memory status before and after the work
        synchronized (optimizeMonitor) {
            try {
                _idxMgr.setDiskIndexerStatus(SearchIndexManager.Status.Working);
                OptimizeScheduler.OptimizeType optType = optScheduler.getScheduledOptimizeType();
                _idxMgr.setPartialExpunge(optType == OptimizeScheduler.OptimizeType.PARTIAL);
                try {
                    super.consume(events);
                } finally {
                    optScheduler.finished();
                    _idxMgr.setPartialExpunge(false);
                }

                if (optType == OptimizeScheduler.OptimizeType.FULL) {
                    try {
                        expungeDeletes();
                    } catch (IOException ioe) {
                        EsearchHealth.setFatal();
                        throw new EsearchException(ioe.getMessage(), ioe);
                    } finally {
                        optScheduler.finished();
                    }
                }
            } finally {
                _idxMgr.setDiskIndexerStatus(SearchIndexManager.Status.Sleep);
            }
        }
    }

    @Override
    public void loadFromIndex(RAMSearchIndex<R> ramIndex) throws EsearchException {
        synchronized (optimizeMonitor) {
            OptimizeScheduler.OptimizeType optType = optScheduler.getScheduledOptimizeType();
            _idxMgr.setPartialExpunge(optType == OptimizeScheduler.OptimizeType.PARTIAL);
            try {
                super.loadFromIndex(ramIndex);
            } finally {
                _idxMgr.setDiskIndexerStatus(SearchIndexManager.Status.Sleep);
                optScheduler.finished();
                _idxMgr.setPartialExpunge(false);
            }

            if (optType == OptimizeScheduler.OptimizeType.FULL) {
                try {
                    expungeDeletes();
                } catch (IOException ioe) {
                    EsearchHealth.setFatal();
                    throw new EsearchException(ioe.getMessage(), ioe);
                } finally {
                    optScheduler.finished();
                }
            }
        }
    }

    public void expungeDeletes() throws IOException {
        log.info("expunging deletes...");
        synchronized (optimizeMonitor) {
            BaseSearchIndex<R> idx = getSearchIndex();
            IndexWriter writer = null;
            try {
                writer = idx.openIndexWriter(_analyzer, _similarity);
                writer.forceMergeDeletes();
            } finally {
                if (writer != null) {
                    idx.closeIndexWriter();
                }
            }
            _idxMgr.refreshDiskReader();
        }
        log.info("deletes expunged");
    }

    public void optimize(int numSegs) throws IOException {
        long t0 = System.currentTimeMillis();
        if (numSegs <= 1) {
            numSegs = 1;
        }
        log.info("optmizing, numSegs: " + numSegs + " ...");

        // we should optimize
        synchronized (optimizeMonitor) {
            BaseSearchIndex<R> idx = getSearchIndex();
            IndexWriter writer = null;
            try {
                writer = idx.openIndexWriter(_analyzer, _similarity);
                writer.forceMerge(numSegs);
            } finally {
                if (writer != null) {
                    idx.closeIndexWriter();
                }
            }
            _idxMgr.refreshDiskReader();
        }
        log.info("index optimized in " + (System.currentTimeMillis() - t0) + "ms");
    }

    public long exportSnapshot(WritableByteChannel channel) throws IOException {
        DiskSearchIndex<R> idx = (DiskSearchIndex<R>) getSearchIndex();
        if (idx != null) {
            DiskIndexSnapshot snapshot = null;

            try {
                synchronized (optimizeMonitor) // prevent index updates while taking a snapshot
                {
                    snapshot = idx.getSnapshot();
                }

                return (snapshot != null ? snapshot.writeTo(channel) : 0);
            } finally {
                if (snapshot != null) {
                    snapshot.close();
                }
            }
        }
        return 0;
    }

    public void importSnapshot(ReadableByteChannel channel) throws IOException {
        DiskSearchIndex<R> idx = (DiskSearchIndex<R>) getSearchIndex();
        if (idx != null) {
            synchronized (optimizeMonitor) // prevent index updates while taking a snapshot
            {
                _idxMgr.purgeIndex();
                idx.importSnapshot(channel);
                _idxMgr.refreshDiskReader();
            }
        }
    }
}