Java tutorial
/* * Copyright (C) 2009 eXo Platform SAS. * * This is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this software; if not, write to the Free * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA * 02110-1301 USA, or see the FSF site: http://www.fsf.org. */ package org.xcmis.search.lucene.index; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.xcmis.search.config.IndexConfiguration; import org.xcmis.search.config.IndexConfigurationException; import org.xcmis.search.lucene.index.merge.AggregatePolicy; import org.xcmis.search.lucene.index.merge.DocumentCountAggregatePolicy; import org.xcmis.search.lucene.index.merge.MaxCandidatsCountAggrigatePolicy; import org.xcmis.search.lucene.index.merge.PendingAggregatePolicy; import org.xcmis.spi.utils.Logger; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Timer; import java.util.TimerTask; /** * Created by The eXo Platform SAS. * * @author <a href="mailto:Sergey.Kabashnyuk@gmail.com">Sergey Kabashnyuk</a> * @version $Id: CacheableIndexDataManager.java 2 2010-02-04 17:21:49Z andrew00x $ */ public class CacheableIndexDataManager extends LocalIndexDataManagerProxy { /** * Data keeper factory. */ private final LuceneIndexDataKeeperFactory dataKeeperFactory; /** * */ private final AggregatePolicy inMemoryAggregationPolicy; /** * Class logger. */ private static final Logger LOG = Logger.getLogger(CacheableIndexDataManager.class); /** * Index chains. */ private final List<LuceneIndexDataManager> memoryChains; /** * */ private final PendingAggregatePolicy persistentAggregationPolicy; /** * The time this index was last flushed or a transaction was committed. */ private volatile long lastFlushTime; /** * Task that is periodically called by {@link #FLUSH_TIMER} and checks if * index should be flushed. */ private TimerTask flushTask; /** * Timer to schedule flushes of this index after some idle time. */ private static final Timer FLUSH_TIMER = new Timer(true); private final static int DEFAULT_IDLE_TIME = 10 * 1000; /** * If manager isStoped */ private boolean isStoped = false; /** * Monitor to use to synchronize access IndexReader. * Extra synchronization to avoid possibility when Timer flashing content to the disc and other * thread call getIndexReader and received null. * */ private final Object updateMonitor = new Object(); /** * @throws IndexConfigurationException * @throws IndexException */ public CacheableIndexDataManager(final IndexConfiguration indexConfuguration) throws IndexException, IndexConfigurationException { super(indexConfuguration); dataKeeperFactory = new InMemoryIndexDataKeeperFactory(); memoryChains = new ArrayList<LuceneIndexDataManager>(); inMemoryAggregationPolicy = new MaxCandidatsCountAggrigatePolicy(new PendingAggregatePolicy()); persistentAggregationPolicy = new PendingAggregatePolicy(); persistentAggregationPolicy.setMaxDirSize(Integer.MAX_VALUE); persistentAggregationPolicy.setMinDirSize(1024 * 1024); persistentAggregationPolicy.setMaxDocuments4Dir(Integer.MAX_VALUE); persistentAggregationPolicy.setMinDocuments4Dir(100); persistentAggregationPolicy.setMinAggregateTime(1 * 1000); persistentAggregationPolicy.setMinModificationTime(3 * 1000); scheduleFlushTask(); } @Override public IndexTransactionModificationReport aggregate(final Collection<LuceneIndexDataManager> indexes) throws IndexException, IndexTransactionException { // dump(); synchronized (indexes) { if (indexes.size() > 2) { final Collection<LuceneIndexDataManager> candidats = inMemoryAggregationPolicy .findIndexDataManagerToAggrigate(indexes, 0, 0); // no candidates to merge if (candidats.size() > 1) { final LuceneIndexDataManager mergedChain = dataKeeperFactory.merge(candidats); for (final LuceneIndexDataManager luceneIndexDataManager : candidats) { dataKeeperFactory.dispose(luceneIndexDataManager); indexes.remove(luceneIndexDataManager); } indexes.add(mergedChain); } } final Collection<LuceneIndexDataManager> candidats2Save = persistentAggregationPolicy .findIndexDataManagerToAggrigate(indexes, 0, 0); if (candidats2Save.size() > 0) { synchronized (updateMonitor) { super.aggregate(candidats2Save); for (final LuceneIndexDataManager luceneIndexDataManager : candidats2Save) { dataKeeperFactory.dispose(luceneIndexDataManager); ((TransactionableLuceneIndexDataManager) luceneIndexDataManager).getTransactionLog() .removeLog(); indexes.remove(luceneIndexDataManager); } lastFlushTime = System.currentTimeMillis(); } } } return null; } @Override public Directory getDirectory() throws IndexException { return super.getDirectory(); } @Override public long getDirectorySize(final boolean includeInherited) { long result = 0; if (memoryChains.size() != 0) { if (includeInherited) { result = super.getDirectorySize(true); } for (final LuceneIndexDataManager dm : memoryChains) { result += dm.getDirectorySize(includeInherited); } } return result; } /** * {@inheritDoc} */ @Override public Document getDocument(final String uuid) throws IndexException { Document doc = null; synchronized (memoryChains) { for (int i = 0; i < memoryChains.size(); i++) { doc = memoryChains.get(i).getDocument(uuid); if (doc != null) { break; } } } if (doc == null) { doc = super.getDocument(uuid); } return doc; } @Override public long getDocumentCount() { long result = super.getDocumentCount(); for (final LuceneIndexDataManager dm : memoryChains) { result += dm.getDocumentCount(); } return result; } /** * {@inheritDoc} * * @throws IndexException */ @Override public IndexReader getIndexReader() throws IndexException { synchronized (memoryChains) { synchronized (updateMonitor) { IndexReader result = super.getIndexReader(); if (memoryChains.size() > 0) { final List<IndexReader> readers = new ArrayList<IndexReader>(memoryChains.size()); final Iterator<LuceneIndexDataManager> it = memoryChains.iterator(); while (it.hasNext()) { final LuceneIndexDataManager chain = it.next(); final IndexReader indexReader = chain.getIndexReader(); if (indexReader != null) { readers.add(indexReader); } } if (result != null) { readers.add(result); } if (readers.size() > 1) { final IndexReader[] indexReaders = new IndexReader[readers.size()]; result = new MultiReader(readers.toArray(indexReaders)); } else if (readers.size() == 1) { result = readers.get(0); } else { throw new IndexReaderNotFoundException("No readers found"); } } if (result == null) { try { RAMDirectory directory = new RAMDirectory(); IndexWriter.MaxFieldLength fieldLength = new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH); IndexWriter iw = new IndexWriter(directory, new SimpleAnalyzer(), true, fieldLength); iw.close(); result = IndexReader.open(directory); } catch (IOException e) { throw new IndexException("Unable to initialize index: empty index "); } } return result; } } } /** * Cancel flush task and add new one */ private void scheduleFlushTask() { // cancel task if (flushTask != null) { flushTask.cancel(); } // clear canceled tasks FLUSH_TIMER.purge(); // new flush task, cause canceled can't be re-used flushTask = new TimerTask() { public void run() { long idleTime = System.currentTimeMillis() - lastFlushTime; //10 sec if (idleTime > DEFAULT_IDLE_TIME) { if (memoryChains.size() > 0) { synchronized (memoryChains) { try { flash(); } catch (TransactionLogException e) { LOG.error(e.getLocalizedMessage(), e); } catch (IndexTransactionException e) { LOG.error(e.getLocalizedMessage(), e); } catch (IndexException e) { LOG.error(e.getLocalizedMessage(), e); } } } } } }; FLUSH_TIMER.schedule(flushTask, 0, 1000); lastFlushTime = System.currentTimeMillis(); } @Override public long getLastModifedTime() { return super.getLastModifedTime(); } /** * {@inheritDoc} */ @Override public IndexTransactionModificationReport save(IndexTransaction<Document> changes) throws IndexException, IndexTransactionException { // notify all chains about changes synchronized (memoryChains) { if (changes.hasModifacationsDocuments()) { // reverse order changes = processModifed(changes); } // if transaction have less then DEFAULT_MAX_DOCUMENTS_4_DIR to add then // create memory index if (DocumentCountAggregatePolicy.DEFAULT_MAX_DOCUMENTS_4_DIR > changes.getAddedDocuments().size()) { changes = changes.apply(processAdded(changes)); } // if transaction have more then DEFAULT_MAX_DOCUMENTS_4_DIR to add or // some modification if (changes.hasAddedDocuments() || changes.hasModifacationsDocuments()) { super.save(changes); } } aggregate(memoryChains); return null; } /** * {@inheritDoc} */ @Override public void stop() { try { if (memoryChains.size() > 0) { synchronized (memoryChains) { flash(); isStoped = true; } } // cancel task if (flushTask != null) { flushTask.cancel(); } // clear canceled tasks FLUSH_TIMER.purge(); FLUSH_TIMER.cancel(); } catch (final ConcurrentModificationException e) { LOG.error(e.getMessage(), e); } catch (final IndexException e) { LOG.error(e.getMessage(), e); } super.stop(); } /** * @throws IndexException * @throws IndexTransactionException * @throws TransactionLogException */ private void flash() throws IndexException, IndexTransactionException, TransactionLogException { if (!isStoped) { synchronized (updateMonitor) { super.aggregate(memoryChains); for (final LuceneIndexDataManager luceneIndexDataManager : memoryChains) { dataKeeperFactory.dispose(luceneIndexDataManager); ((TransactionableLuceneIndexDataManager) luceneIndexDataManager).getTransactionLog() .removeLog(); } memoryChains.clear(); lastFlushTime = System.currentTimeMillis(); } } } private void dump() { LOG.info("====" + memoryChains.size() + "====="); for (final LuceneIndexDataManager luceneIndexDataManager : memoryChains) { LOG.info(luceneIndexDataManager.getDirectorySize(false) + "\t\t" + luceneIndexDataManager.getDocumentCount() + "\t\t" + (System.currentTimeMillis() - luceneIndexDataManager.getLastModifedTime()) + " msec"); } } /** * Process add * * @param changes * @throws IndexException */ private IndexTransactionModificationReportImpl processAdded(final IndexTransaction<Document> changes) throws IndexException { if (changes.getAddedDocuments().size() > 0) { final LuceneIndexDataManager indexDataKeeper = dataKeeperFactory.createNewIndexDataKeeper(changes); indexDataKeeper.start(); synchronized (memoryChains) { memoryChains.add(indexDataKeeper); } } return new IndexTransactionModificationReportImpl(changes.getAddedDocuments().keySet(), new HashSet<String>(), new HashSet<String>()); } /** * Process remove and update * * @param changes * @return * @throws IndexException * @throws IndexTransactionException */ private IndexTransaction<Document> processModifed(IndexTransaction<Document> changes) throws IndexException, IndexTransactionException { synchronized (memoryChains) { for (final Iterator<LuceneIndexDataManager> it = memoryChains.iterator(); it.hasNext();) { final LuceneIndexDataManager chain = it.next(); final IndexTransactionModificationReport report = chain.save(changes); if (report.isModifed()) { changes = changes.apply(report); if (chain.getDocumentCount() == 0) { dataKeeperFactory.dispose(chain); ((TransactionableLuceneIndexDataManager) chain).getTransactionLog().removeLog(); it.remove(); } } if (!changes.hasModifacationsDocuments()) { break; } } } return changes; } }