List of usage examples for org.apache.solr.search SolrIndexSearcher warm
public void warm(SolrIndexSearcher old)
From source file:org.alfresco.solr.AlfrescoSolrEventListener.java
License:Open Source License
@Override public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) { Properties p = core.getResourceLoader().getCoreProperties(); boolean doPermissionChecks = Boolean.parseBoolean(p.getProperty("alfresco.doPermissionChecks", "true")); SolrIndexReader newReader = newSearcher.getReader(); log.info("Max " + newReader.maxDoc()); log.info("Docs " + newReader.numDocs()); log.info("Deleted " + newReader.numDeletedDocs()); long startTime = System.nanoTime(); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_DOC_ID); indexedByDocId.resize(newReader.maxDoc()); HashSet<String> globalReaders = new HashSet<String>(); OpenBitSet allLeafDocs = new OpenBitSet(newReader.maxDoc()); long[] aclIdByDocId = new long[newReader.maxDoc()]; long[] txByDocId = new long[newReader.maxDoc()]; long[] aclTxByDocId = new long[newReader.maxDoc()]; for (int i = 0; i < aclIdByDocId.length; i++) { aclIdByDocId[i] = -1;//from w w w . j ava2 s . c o m txByDocId[i] = -1; aclTxByDocId[i] = -1; } OpenBitSet deleted = new OpenBitSet(newReader.maxDoc()); OwnerIdManager ownerIdManager = new OwnerIdManager(); HashMap<Long, CacheEntry> unmatchedByDBID = new HashMap<Long, CacheEntry>(); if ((incrementalCacheRebuild) && currentSearcher != null) { ResizeableArrayList<CacheEntry> oldIndexedByDocId = (ResizeableArrayList<CacheEntry>) currentSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_DOC_ID); long[] oldAclIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ACL_ID_BY_DOC_ID); long[] oldTxIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_TX_ID_BY_DOC_ID); long[] oldAclTxIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ACL_TX_ID_BY_DOC_ID); OpenBitSet oldAllLeafDocs = (OpenBitSet) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ALL_LEAF_DOCS); OwnerIdManager oldOwnerIdManager = (OwnerIdManager) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_OWNER_ID_MANAGER); ownerIdManager.addAll(oldOwnerIdManager); ConcurrentHashMap<Long, Long> addedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_LEAVES); ConcurrentHashMap<Long, Long> addedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_AUX); ConcurrentHashMap<Long, Long> addedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_ACL); ConcurrentHashMap<Long, Long> addedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_TX); ConcurrentHashMap<Long, Long> addedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_ACL_TX); ConcurrentHashMap<Long, Long> updatedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_LEAVES); ConcurrentHashMap<Long, Long> updatedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_AUX); ConcurrentHashMap<Long, Long> updatedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_ACL); ConcurrentHashMap<Long, Long> updatedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_TX); ConcurrentHashMap<Long, Long> updatedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_ACL_TX); ConcurrentHashMap<Long, Long> deletedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_LEAVES); ConcurrentHashMap<Long, Long> deletedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_AUX); ConcurrentHashMap<Long, Long> deletedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_ACL); ConcurrentHashMap<Long, Long> deletedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_TX); ConcurrentHashMap<Long, Long> deletedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_ACL_TX); AtomicBoolean deleteAll = (AtomicBoolean) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_DELETE_ALL); AtomicBoolean checkCache = (AtomicBoolean) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_CHECK_CACHE); if (checkCache == null) { checkCache = new AtomicBoolean(false); } boolean hasNew = (addedLeaves.size() + addedAux.size() + addedAcl.size() + addedTx.size() + addedAclTx.size() + updatedLeaves.size() + updatedAux.size() + updatedAcl.size() + updatedTx.size() + updatedAclTx.size()) > 0; if (newReader.maxDoc() == 0) { // nothing to do } else if ((oldIndexedByDocId == null) || (oldAclIdByDocId == null) || (oldTxIdByDocId == null) || (oldAclTxIdByDocId == null) || (oldAllLeafDocs == null) || (oldOwnerIdManager == null)) { log.warn("Recover from missing cache"); buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), unmatchedByDBID, ownerIdManager); } else if (deleteAll.get()) { buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), unmatchedByDBID, ownerIdManager); } else { try { SolrIndexReader[] before = currentSearcher.getReader().getSequentialSubReaders(); SolrIndexReader[] after = newSearcher.getReader().getSequentialSubReaders(); CacheSection[] cacheSectionsBefore = SolrIndexReaderCacheSection.getCacheSections(before); CacheSection[] cacheSectionsAfter = SolrIndexReaderCacheSection.getCacheSections(after); // Copy old to new and apply deletions int currentCache = 0; for (int i = 0; i < oldAclIdByDocId.length; i++) { CacheSection section = cacheSectionsBefore[currentCache]; if (section.getStart() + section.getLength() == i) { currentCache++; if (currentCache == cacheSectionsBefore.length) { currentCache--; } section = cacheSectionsBefore[currentCache]; } CacheEntry entry = oldIndexedByDocId.get(i); if (entry != null) { if (entry.getLeaf() == i) { // Leaf if ((updatedLeaves.get(entry.dbid) == null) && (deletedLeaves.get(entry.dbid) == null)) { // leave } else { section.addDeletion(i); deleted.set(i); } } else if (entry.getPath() == i) { // Aux if ((updatedAux.get(entry.dbid) == null) && (deletedAux.get(entry.dbid) == null)) { // leave } else { section.addDeletion(i); deleted.set(i); } } } else { if ((updatedAcl.get(oldAclIdByDocId[i]) != null) || (deletedAcl.get(oldAclIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } if ((updatedTx.get(oldTxIdByDocId[i]) != null) || (deletedTx.get(oldTxIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } if ((updatedAclTx.get(oldAclTxIdByDocId[i]) != null) || (deletedAclTx.get(oldAclTxIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } } } LinkedList<CacheMatch> operations = buildCacheUpdateOperations(hasNew, cacheSectionsBefore, cacheSectionsAfter, after); log.info("Cache operatoins ..."); for (CacheMatch match : operations) { log.info(match.toString()); } CacheUpdateTracker tracker = new CacheUpdateTracker(0, 0, 0); for (CacheMatch match : operations) { match.updateCache(tracker, oldIndexedByDocId, oldAclIdByDocId, oldTxIdByDocId, oldAclTxIdByDocId, indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, unmatchedByDBID, deleted, newReader, ownerIdManager); } // Check unmatched int hiddenDocCount = 0; for (Long unmatchedDBID : unmatchedByDBID.keySet()) { // hidden docs appear as an unmatched path CacheEntry entry = unmatchedByDBID.get(unmatchedDBID); if ((entry.getLeaf() != 0) && (entry.getPath() == 0)) { // leaf doc with no aux doc; log.info("Leaf has no AUX doc for DBID " + unmatchedDBID + " at position " + entry.getLeaf()); } if ((entry.getLeaf() == 0) && (entry.getPath() != 0)) { hiddenDocCount++; } } log.info("Cache unindexed/error doc count = " + hiddenDocCount); // Simple position check; boolean simpleCheckOk = true; for (int i = 0; i < indexedByDocId.size(); i++) { CacheEntry entry = indexedByDocId.get(i); if (entry != null) { if ((entry.getLeaf() != i) && (entry.getPath() != i)) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Simple cache caheck failed: Incorrect indexedByDocId at " + i); log.warn(".. leaf and path doc poistion do not match the doc position .. " + indexedByDocId.get(i)); simpleCheckOk = false; break; } } } if ((simpleCheckOk == false) || forceCheckCache || checkCache.get()) { ResizeableArrayList<CacheEntry> checkIndexedByDocId = (ResizeableArrayList<CacheEntry>) currentSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_CHECK_INDEXED_BY_DOC_ID_LIST); checkIndexedByDocId.resize(newReader.maxDoc()); OpenBitSet checkAllLeafDocs = new OpenBitSet(newReader.maxDoc()); long[] checkAclIdByDocId = new long[newReader.maxDoc()]; long[] checkTxIdByDocId = new long[newReader.maxDoc()]; long[] checkAclTxIdByDocId = new long[newReader.maxDoc()]; buildCacheForReader(checkIndexedByDocId, checkAllLeafDocs, checkAclIdByDocId, checkTxIdByDocId, checkAclTxIdByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); boolean ok = true; boolean thisTestOk = true; for (int i = 0; i < checkIndexedByDocId.size(); i++) { if (!EqualsHelper.nullSafeEquals(checkIndexedByDocId.get(i), indexedByDocId.get(i))) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid indexedByDocId at " + i); log.warn(".. found .. " + indexedByDocId.get(i)); log.warn(".. expected .. " + checkIndexedByDocId.get(i)); ok = false; thisTestOk = false; } } } thisTestOk = true; if (!checkAllLeafDocs.equals(allLeafDocs)) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid AllLeafDocs cache"); ok = false; thisTestOk = false; } } thisTestOk = true; for (int i = 0; i < checkAclIdByDocId.length; i++) { if (checkAclIdByDocId[i] != aclIdByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid AclIdByDocId cache at " + i); log.warn(".. found .. " + aclIdByDocId[i]); log.warn(".. expected .. " + checkAclIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } thisTestOk = true; for (int i = 0; i < checkTxIdByDocId.length; i++) { if (checkTxIdByDocId[i] != txByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid txByDocId cache at " + i); log.warn(".. found .. " + txByDocId[i]); log.warn(".. expected .. " + checkTxIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } thisTestOk = true; for (int i = 0; i < checkAclTxIdByDocId.length; i++) { if (checkAclTxIdByDocId[i] != aclTxByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid aclTxByDocId cache at " + i); log.warn(".. found .. " + aclTxByDocId[i]); log.warn(".. expected .. " + checkAclTxIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } if (!ok) { indexedByDocId.copyFrom(checkIndexedByDocId); allLeafDocs = checkAllLeafDocs; aclIdByDocId = checkAclIdByDocId; txByDocId = checkTxIdByDocId; aclTxByDocId = checkAclTxIdByDocId; log.warn("... Using recomputed cache"); } else { log.info("... cache OK"); } } } catch (IllegalStateException ise) { log.info("Cache state error -> rebuilding", ise); buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); } } } else { buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); } long endTime = System.nanoTime(); log.info("Core cache rebuilt in " + ((endTime - startTime) / (1.0e9))); startTime = System.nanoTime(); int size = doPermissionChecks ? (int) allLeafDocs.cardinality() : 0; ResizeableArrayList<CacheEntry> indexedOderedByAclIdThenDoc = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_ACL_ID_THEN_LEAF); indexedOderedByAclIdThenDoc.resize(size); ResizeableArrayList<CacheEntry> indexedOderedByOwnerIdThenDoc = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_OWNER_ID_THEN_LEAF); indexedOderedByOwnerIdThenDoc.resize(size); if (doPermissionChecks) { int doc = -1; int pos = 0; while ((doc = allLeafDocs.nextSetBit(doc + 1)) != -1) { CacheEntry entry = indexedByDocId.get(doc); indexedOderedByAclIdThenDoc.set(pos, entry); indexedOderedByOwnerIdThenDoc.set(pos, entry); pos++; } } indexedOderedByAclIdThenDoc.sort(new Comparator<CacheEntry>() { @Override public int compare(CacheEntry o1, CacheEntry o2) { if (o2 == null) { if (o1 == null) { return 0; } else { return -1; // nulls at the end } } else { if (o1 == null) { return 1; } else { long diff = o1.getAclid() - o2.getAclid(); if (diff == 0L) { return o1.getLeaf() - o2.getLeaf(); } else { return (diff > 0L) ? 1 : -1; } } } } }); // build lookups HashMap<AclLookUp, AclLookUp> aclLookUp = new HashMap<AclLookUp, AclLookUp>(); AclLookUp currentAclLookUp = null; for (int i = 0; i < indexedOderedByAclIdThenDoc.size(); i++) { CacheEntry entry = indexedOderedByAclIdThenDoc.get(i); if (entry != null) { if (currentAclLookUp == null) { currentAclLookUp = new AclLookUp(entry.getAclid(), i); } else { if (currentAclLookUp.aclid == entry.aclid) { // carry on } else { // acl id has changed - new set currentAclLookUp.setEnd(i); AclLookUp next = new AclLookUp(entry.getAclid(), i); aclLookUp.put(currentAclLookUp, currentAclLookUp); currentAclLookUp = next; } } } else { // found first null we are done if (currentAclLookUp != null) { currentAclLookUp.setEnd(i); aclLookUp.put(currentAclLookUp, currentAclLookUp); } break; } } if (currentAclLookUp != null) { currentAclLookUp.setEnd(indexedOderedByAclIdThenDoc.size()); aclLookUp.put(currentAclLookUp, currentAclLookUp); } indexedOderedByOwnerIdThenDoc.sort(new Comparator<CacheEntry>() { @Override public int compare(CacheEntry o1, CacheEntry o2) { if (o2 == null) { if (o1 == null) { return 0; } else { return -1; // nulls at the end } } else { if (o1 == null) { return 1; } else { int diff = o1.getOwner() - o2.getOwner(); if (diff == 0) { return o1.getLeaf() - o2.getLeaf(); } else { return diff; } } } } }); // build lookups HashMap<String, OwnerLookUp> ownerLookUp = new HashMap<String, OwnerLookUp>(); OwnerLookUp currentOwnerLookUp = null; for (int i = 0; i < indexedOderedByOwnerIdThenDoc.size(); i++) { CacheEntry entry = indexedOderedByOwnerIdThenDoc.get(i); if (entry != null) { if (currentOwnerLookUp == null) { currentOwnerLookUp = new OwnerLookUp(entry.getOwner(), i); } else { if (currentOwnerLookUp.owner == entry.owner) { // carry on } else { // acl id has changed - new set currentOwnerLookUp.setEnd(i); OwnerLookUp next = new OwnerLookUp(entry.getOwner(), i); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } currentOwnerLookUp = next; } } } else { // found first null we are done if (currentOwnerLookUp != null) { currentOwnerLookUp.setEnd(i); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } } break; } } if (currentOwnerLookUp != null) { currentOwnerLookUp.setEnd(indexedOderedByOwnerIdThenDoc.size()); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } } // cache readers and acl doc ids //HashMap<String, HashSet<Long>> readerToAclIds = new HashMap<String, HashSet<Long>>(); BitDocSet publicDocSet = new BitDocSet(new OpenBitSet(newReader.maxDoc())); if (doPermissionChecks) { try { HashSet<Long> globallyReadableAcls = buildReaderAclIds(newSearcher, "GROUP_EVERYONE", aclIdByDocId); newSearcher.cacheInsert(AlfrescoSolrEventListener.ALFRESCO_READER_TO_ACL_IDS_CACHE, "GROUP_EVERYONE", globallyReadableAcls); AclLookUp key = new AclLookUp(0); for (Long longAcl : globallyReadableAcls) { key.setAclid(longAcl); AlfrescoSolrEventListener.AclLookUp value = aclLookUp.get(key); if (value != null) { for (int i = value.getStart(); i < value.getEnd(); i++) { publicDocSet.add(indexedOderedByAclIdThenDoc.get(i).getLeaf()); } } } } catch (IOException e) { log.error("IO Exception while warming searcher", e); } } // transform to readers to acl ids endTime = System.nanoTime(); log.info("Derived caches rebuilt in " + ((endTime - startTime) / (1.0e9))); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_ID_BY_DOC_ID, aclIdByDocId); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_TX_ID_BY_DOC_ID, txByDocId); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_TX_ID_BY_DOC_ID, aclTxByDocId); // TODO: Make global readers configurable. globalReaders.add(PermissionService.OWNER_AUTHORITY); globalReaders.add(PermissionService.ADMINISTRATOR_AUTHORITY); globalReaders.add(AuthenticationUtil.getSystemUserName()); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_GLOBAL_READERS, globalReaders); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ALL_LEAF_DOCS, allLeafDocs); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_LOOKUP, aclLookUp); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_OWNER_LOOKUP, ownerLookUp); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_OWNER_ID_MANAGER, ownerIdManager); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_PUBLIC_DOC_SET, publicDocSet); try { if (currentSearcher != null) { newSearcher.warm(currentSearcher); } } catch (IOException e) { log.error("IO Exception while warming searcher", e); } }