Java tutorial
/***************************************************************************** * * Copyright (C) Zenoss, Inc. 2011, all rights reserved. * * This content is made available according to terms specified in * License.zenoss under the directory where your Zenoss product is installed. * ****************************************************************************/ package org.zenoss.zep.index.impl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; import org.zenoss.protobufs.zep.Zep.EventDetailItem; import org.zenoss.protobufs.zep.Zep.EventSummary; import org.zenoss.protobufs.zep.Zep.ZepConfig; import org.zenoss.zep.ZepException; import org.zenoss.zep.dao.*; import org.zenoss.zep.dao.impl.DaoUtils; import org.zenoss.zep.events.IndexRebuildRequiredEvent; import org.zenoss.zep.impl.ThreadRenamingRunnable; import org.zenoss.zep.index.EventIndexDao; import org.zenoss.zep.index.EventIndexRebuilder; import org.zenoss.zep.index.EventIndexer; import org.zenoss.zep.index.IndexedDetailsConfiguration; import java.io.File; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** * Utility used to rebuild the event index. */ public class EventIndexRebuilderImpl implements EventIndexRebuilder, ApplicationListener<IndexRebuildRequiredEvent> { private static final Logger logger = LoggerFactory.getLogger(EventIndexRebuilderImpl.class); private final boolean enableIndexing; private EventIndexer eventIndexer; private ConfigDao configDao; private EventSummaryBaseDao summaryBaseDao; private EventIndexDao indexDao; private IndexMetadataDao indexMetadataDao; private IndexedDetailsConfiguration indexedDetailsConfiguration; private final ExecutorService executorService = Executors.newSingleThreadExecutor(); private Future<?> rebuildFuture = null; private volatile boolean shutdown = false; private volatile boolean configurationChanged = false; private final Object lock = new Object(); private byte[] indexVersionHash; private File indexStateFile; public EventIndexRebuilderImpl(boolean enableIndexing) { this.enableIndexing = enableIndexing; } @Override public void init() { if (!enableIndexing) { return; } rebuildFuture = executorService.submit(new ThreadRenamingRunnable(new Runnable() { @Override public void run() { logger.info("Index rebuilding thread started for: {}", indexDao.getName()); while (!shutdown) { try { configurationChanged = false; Map<String, EventDetailItem> detailItems = indexedDetailsConfiguration .getEventDetailItemsByName(); indexVersionHash = calculateIndexVersionHash(detailItems); recreateIndexIfNeeded(); // Wait to be interrupted if the configuration changes for the index synchronized (lock) { if (!shutdown && !configurationChanged) { lock.wait(); } } } catch (InterruptedException e) { logger.info("Interrupted index rebuilding thread"); } catch (Exception e) { logger.warn("Failed to rebuild event index", e); /* * If we got an error indexing or accessing the database, back off a bit and wait for any * transient errors to be resolved. */ synchronized (lock) { try { lock.wait(30000L); } catch (InterruptedException ie) { // Ignored } } } } logger.info("Index rebuilding thread stopped for: {}", indexDao.getName()); } }, "INDEX_REBUILDER_" + this.indexDao.getName().toUpperCase())); } public void setIndexDir(File indexDir) { this.indexStateFile = new File(indexDir, ".index_state_" + indexDao.getName() + ".properties"); if (this.enableIndexing) { File parentDir = this.indexStateFile.getParentFile(); if (!parentDir.isDirectory() && !parentDir.mkdirs()) { logger.warn("Failed to create parent directory: {}", parentDir.getAbsolutePath()); } } } public void setEventIndexer(EventIndexer eventIndexer) { this.eventIndexer = eventIndexer; } public void setConfigDao(final ConfigDao configDao) { this.configDao = configDao; } public void setSummaryBaseDao(EventSummaryBaseDao summaryBaseDao) { this.summaryBaseDao = summaryBaseDao; } public void setIndexDao(EventIndexDao indexDao) { this.indexDao = indexDao; } public void setIndexMetadataDao(IndexMetadataDao indexMetadataDao) { this.indexMetadataDao = indexMetadataDao; } public void setIndexedDetailsConfiguration(IndexedDetailsConfiguration indexedDetailsConfiguration) { this.indexedDetailsConfiguration = indexedDetailsConfiguration; } private void deleteStateFile() { if (this.indexStateFile.isFile() && !this.indexStateFile.delete()) { logger.info("Failed to remove index rebuild state file"); } } private static byte[] calculateIndexVersionHash(Map<String, EventDetailItem> detailItems) throws ZepException { TreeMap<String, EventDetailItem> sorted = new TreeMap<String, EventDetailItem>(detailItems); StringBuilder indexConfigStr = new StringBuilder(); for (EventDetailItem item : sorted.values()) { // Only key and type affect the indexing behavior - ignore changes to display name indexConfigStr.append('|'); indexConfigStr.append(item.getKey()); indexConfigStr.append('|'); indexConfigStr.append(item.getType().name()); indexConfigStr.append('|'); } if (indexConfigStr.length() == 0) { return null; } return DaoUtils.sha1(indexConfigStr.toString()); } public void shutdown() throws InterruptedException { this.shutdown = true; synchronized (this.lock) { this.lock.notifyAll(); } if (rebuildFuture != null) { try { rebuildFuture.get(); } catch (ExecutionException e) { logger.warn("Error rebuilding event index", e); } } this.executorService.shutdown(); this.executorService.awaitTermination(0, TimeUnit.SECONDS); } private void recreateIndexIfNeeded() throws ZepException, InterruptedException { final IndexMetadata indexMetadata = indexMetadataDao.findIndexMetadata(indexDao.getName()); final int numDocs = indexDao.getNumDocs(); boolean recreateIndex = false; IndexRebuildState indexRebuildState = null; // Stop the event indexer if it is currently running. eventIndexer.stop(); // Recreate index if we detect an empty index (could have been wiped from disk) - This check only happens once if (numDocs == 0) { logger.info("Empty index detected."); deleteStateFile(); recreateIndex = true; } // Recreate index if we detect that the schema was cleared (wiped record of indexing state) else if (indexMetadata == null) { if (numDocs > 0) { logger.info("Inconsistent state between index and database. Clearing index."); indexDao.clear(); } deleteStateFile(); recreateIndex = true; } // Recreate index if the version number or indexed details changed else { recreateIndex = false; try { indexRebuildState = IndexRebuildState.loadState(this.indexStateFile); } catch (Exception e) { logger.warn("Failed to restore index rebuild state from: " + this.indexStateFile.getAbsolutePath(), e); } if (indexRebuildState != null) { recreateIndex = true; if (indexVersionChanged(indexMetadata)) { if (indexRebuildState.getIndexVersion() != IndexConstants.INDEX_VERSION || !Arrays.equals(indexRebuildState.getIndexVersionHash(), this.indexVersionHash)) { // We have state from an previous version / hash - ignore it indexRebuildState = null; deleteStateFile(); } } } } if (recreateIndex) { /* * We add dummy index metadata here to ensure that if ZEP is stopped while rebuilding the index, we will be * able to restart the indexing process. */ byte[] checksum = new byte[20]; Arrays.fill(checksum, (byte) 0); this.indexMetadataDao.updateIndexVersion(this.indexDao.getName(), 0, checksum); // We want to start the event indexer before we start the rebuild (we want both to run in parallel). eventIndexer.start(this.configDao.getConfig()); recreateIndexFromDatabase(indexRebuildState); } else { // Start the event indexer - we have done all the necessary initialization. eventIndexer.start(this.configDao.getConfig()); } } private boolean indexVersionChanged(IndexMetadata indexMetadata) { boolean changed = false; if (IndexConstants.INDEX_VERSION != indexMetadata.getIndexVersion()) { logger.info("Index version changed: previous={}, new={}", indexMetadata.getIndexVersion(), IndexConstants.INDEX_VERSION); changed = true; } else if (!Arrays.equals(this.indexVersionHash, indexMetadata.getIndexVersionHash())) { logger.info("Index configuration changed."); changed = true; } return changed; } private void recreateIndexFromDatabase(IndexRebuildState indexRebuildState) throws ZepException { logger.info("Recreating index for table {}", indexDao.getName()); EventBatchParams nextBatch = null; final long throughTime; if (indexRebuildState == null) { throughTime = System.currentTimeMillis(); indexRebuildState = new IndexRebuildState(IndexConstants.INDEX_VERSION, this.indexVersionHash, throughTime, null, null); } else { Long startingLastSeen = indexRebuildState.getStartingLastSeen(); String startingUuid = indexRebuildState.getStartingUuid(); if (startingLastSeen != null || startingUuid != null) { nextBatch = new EventBatchParams(startingLastSeen, startingUuid); } throughTime = indexRebuildState.getThroughTime(); logger.info("Resuming event indexing from: {}", nextBatch); } int i = 0; int numIndexed = 0; List<EventSummary> events; do { ZepConfig config = configDao.getConfig(); logger.debug("About to configDao.listBatch"); EventBatch batch = summaryBaseDao.listBatch(nextBatch, throughTime, config.getIndexLimit()); events = batch.events; nextBatch = batch.nextParams; i++; logger.debug("About to indexDao.indexMany"); indexDao.indexMany(events); logger.debug("Finished indexDao.indexMany"); numIndexed += events.size(); if (this.configurationChanged || this.shutdown) { break; } indexDao.commit(); //TODO: perhaps remove this indexRebuildState.setStartingUuid(nextBatch.nextUuid); indexRebuildState.setStartingLastSeen(nextBatch.nextLastSeen); if (i % 25 == 0) { logger.info("Indexed {} events on table {}", numIndexed, indexDao.getName()); } try { indexRebuildState.save(this.indexStateFile); } catch (Exception e) { logger.warn( "Failed to save state of index rebuild to file: " + this.indexStateFile.getAbsolutePath(), e); } } while (nextBatch.nextUuid != null && !this.configurationChanged && !this.shutdown); if (this.configurationChanged || this.shutdown) { logger.info("Index rebuild aborted"); return; } if (numIndexed > 0) { indexDao.commit(); indexMetadataDao.updateIndexVersion(indexDao.getName(), IndexConstants.INDEX_VERSION, indexVersionHash); } logger.info("Finished recreating index for {} events on table: {}", numIndexed, indexDao.getName()); deleteStateFile(); } @Override public void onApplicationEvent(IndexRebuildRequiredEvent event) { this.configurationChanged = true; synchronized (this.lock) { this.lock.notifyAll(); } } }