List of usage examples for org.apache.solr.search SolrIndexSearcher cacheLookup
public Object cacheLookup(String cacheName, Object key)
From source file:org.alfresco.solr.AlfrescoSolrEventListener.java
License:Open Source License
@Override public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) { Properties p = core.getResourceLoader().getCoreProperties(); boolean doPermissionChecks = Boolean.parseBoolean(p.getProperty("alfresco.doPermissionChecks", "true")); SolrIndexReader newReader = newSearcher.getReader(); log.info("Max " + newReader.maxDoc()); log.info("Docs " + newReader.numDocs()); log.info("Deleted " + newReader.numDeletedDocs()); long startTime = System.nanoTime(); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_DOC_ID); indexedByDocId.resize(newReader.maxDoc()); HashSet<String> globalReaders = new HashSet<String>(); OpenBitSet allLeafDocs = new OpenBitSet(newReader.maxDoc()); long[] aclIdByDocId = new long[newReader.maxDoc()]; long[] txByDocId = new long[newReader.maxDoc()]; long[] aclTxByDocId = new long[newReader.maxDoc()]; for (int i = 0; i < aclIdByDocId.length; i++) { aclIdByDocId[i] = -1;/*from w w w.ja va 2 s .co m*/ txByDocId[i] = -1; aclTxByDocId[i] = -1; } OpenBitSet deleted = new OpenBitSet(newReader.maxDoc()); OwnerIdManager ownerIdManager = new OwnerIdManager(); HashMap<Long, CacheEntry> unmatchedByDBID = new HashMap<Long, CacheEntry>(); if ((incrementalCacheRebuild) && currentSearcher != null) { ResizeableArrayList<CacheEntry> oldIndexedByDocId = (ResizeableArrayList<CacheEntry>) currentSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_DOC_ID); long[] oldAclIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ACL_ID_BY_DOC_ID); long[] oldTxIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_TX_ID_BY_DOC_ID); long[] oldAclTxIdByDocId = (long[]) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ACL_TX_ID_BY_DOC_ID); OpenBitSet oldAllLeafDocs = (OpenBitSet) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_ALL_LEAF_DOCS); OwnerIdManager oldOwnerIdManager = (OwnerIdManager) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_OWNER_ID_MANAGER); ownerIdManager.addAll(oldOwnerIdManager); ConcurrentHashMap<Long, Long> addedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_LEAVES); ConcurrentHashMap<Long, Long> addedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_AUX); ConcurrentHashMap<Long, Long> addedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_ACL); ConcurrentHashMap<Long, Long> addedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_TX); ConcurrentHashMap<Long, Long> addedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_ADDED_ACL_TX); ConcurrentHashMap<Long, Long> updatedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_LEAVES); ConcurrentHashMap<Long, Long> updatedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_AUX); ConcurrentHashMap<Long, Long> updatedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_ACL); ConcurrentHashMap<Long, Long> updatedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_TX); ConcurrentHashMap<Long, Long> updatedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_UPDATED_ACL_TX); ConcurrentHashMap<Long, Long> deletedLeaves = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_LEAVES); ConcurrentHashMap<Long, Long> deletedAux = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_AUX); ConcurrentHashMap<Long, Long> deletedAcl = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_ACL); ConcurrentHashMap<Long, Long> deletedTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_TX); ConcurrentHashMap<Long, Long> deletedAclTx = (ConcurrentHashMap<Long, Long>) currentSearcher .cacheLookup(ALFRESCO_CACHE, KEY_DELETED_ACL_TX); AtomicBoolean deleteAll = (AtomicBoolean) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_DELETE_ALL); AtomicBoolean checkCache = (AtomicBoolean) currentSearcher.cacheLookup(ALFRESCO_CACHE, KEY_CHECK_CACHE); if (checkCache == null) { checkCache = new AtomicBoolean(false); } boolean hasNew = (addedLeaves.size() + addedAux.size() + addedAcl.size() + addedTx.size() + addedAclTx.size() + updatedLeaves.size() + updatedAux.size() + updatedAcl.size() + updatedTx.size() + updatedAclTx.size()) > 0; if (newReader.maxDoc() == 0) { // nothing to do } else if ((oldIndexedByDocId == null) || (oldAclIdByDocId == null) || (oldTxIdByDocId == null) || (oldAclTxIdByDocId == null) || (oldAllLeafDocs == null) || (oldOwnerIdManager == null)) { log.warn("Recover from missing cache"); buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), unmatchedByDBID, ownerIdManager); } else if (deleteAll.get()) { buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), unmatchedByDBID, ownerIdManager); } else { try { SolrIndexReader[] before = currentSearcher.getReader().getSequentialSubReaders(); SolrIndexReader[] after = newSearcher.getReader().getSequentialSubReaders(); CacheSection[] cacheSectionsBefore = SolrIndexReaderCacheSection.getCacheSections(before); CacheSection[] cacheSectionsAfter = SolrIndexReaderCacheSection.getCacheSections(after); // Copy old to new and apply deletions int currentCache = 0; for (int i = 0; i < oldAclIdByDocId.length; i++) { CacheSection section = cacheSectionsBefore[currentCache]; if (section.getStart() + section.getLength() == i) { currentCache++; if (currentCache == cacheSectionsBefore.length) { currentCache--; } section = cacheSectionsBefore[currentCache]; } CacheEntry entry = oldIndexedByDocId.get(i); if (entry != null) { if (entry.getLeaf() == i) { // Leaf if ((updatedLeaves.get(entry.dbid) == null) && (deletedLeaves.get(entry.dbid) == null)) { // leave } else { section.addDeletion(i); deleted.set(i); } } else if (entry.getPath() == i) { // Aux if ((updatedAux.get(entry.dbid) == null) && (deletedAux.get(entry.dbid) == null)) { // leave } else { section.addDeletion(i); deleted.set(i); } } } else { if ((updatedAcl.get(oldAclIdByDocId[i]) != null) || (deletedAcl.get(oldAclIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } if ((updatedTx.get(oldTxIdByDocId[i]) != null) || (deletedTx.get(oldTxIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } if ((updatedAclTx.get(oldAclTxIdByDocId[i]) != null) || (deletedAclTx.get(oldAclTxIdByDocId[i]) != null)) { section.addDeletion(i); deleted.set(i); } } } LinkedList<CacheMatch> operations = buildCacheUpdateOperations(hasNew, cacheSectionsBefore, cacheSectionsAfter, after); log.info("Cache operatoins ..."); for (CacheMatch match : operations) { log.info(match.toString()); } CacheUpdateTracker tracker = new CacheUpdateTracker(0, 0, 0); for (CacheMatch match : operations) { match.updateCache(tracker, oldIndexedByDocId, oldAclIdByDocId, oldTxIdByDocId, oldAclTxIdByDocId, indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, unmatchedByDBID, deleted, newReader, ownerIdManager); } // Check unmatched int hiddenDocCount = 0; for (Long unmatchedDBID : unmatchedByDBID.keySet()) { // hidden docs appear as an unmatched path CacheEntry entry = unmatchedByDBID.get(unmatchedDBID); if ((entry.getLeaf() != 0) && (entry.getPath() == 0)) { // leaf doc with no aux doc; log.info("Leaf has no AUX doc for DBID " + unmatchedDBID + " at position " + entry.getLeaf()); } if ((entry.getLeaf() == 0) && (entry.getPath() != 0)) { hiddenDocCount++; } } log.info("Cache unindexed/error doc count = " + hiddenDocCount); // Simple position check; boolean simpleCheckOk = true; for (int i = 0; i < indexedByDocId.size(); i++) { CacheEntry entry = indexedByDocId.get(i); if (entry != null) { if ((entry.getLeaf() != i) && (entry.getPath() != i)) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Simple cache caheck failed: Incorrect indexedByDocId at " + i); log.warn(".. leaf and path doc poistion do not match the doc position .. " + indexedByDocId.get(i)); simpleCheckOk = false; break; } } } if ((simpleCheckOk == false) || forceCheckCache || checkCache.get()) { ResizeableArrayList<CacheEntry> checkIndexedByDocId = (ResizeableArrayList<CacheEntry>) currentSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_CHECK_INDEXED_BY_DOC_ID_LIST); checkIndexedByDocId.resize(newReader.maxDoc()); OpenBitSet checkAllLeafDocs = new OpenBitSet(newReader.maxDoc()); long[] checkAclIdByDocId = new long[newReader.maxDoc()]; long[] checkTxIdByDocId = new long[newReader.maxDoc()]; long[] checkAclTxIdByDocId = new long[newReader.maxDoc()]; buildCacheForReader(checkIndexedByDocId, checkAllLeafDocs, checkAclIdByDocId, checkTxIdByDocId, checkAclTxIdByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); boolean ok = true; boolean thisTestOk = true; for (int i = 0; i < checkIndexedByDocId.size(); i++) { if (!EqualsHelper.nullSafeEquals(checkIndexedByDocId.get(i), indexedByDocId.get(i))) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid indexedByDocId at " + i); log.warn(".. found .. " + indexedByDocId.get(i)); log.warn(".. expected .. " + checkIndexedByDocId.get(i)); ok = false; thisTestOk = false; } } } thisTestOk = true; if (!checkAllLeafDocs.equals(allLeafDocs)) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid AllLeafDocs cache"); ok = false; thisTestOk = false; } } thisTestOk = true; for (int i = 0; i < checkAclIdByDocId.length; i++) { if (checkAclIdByDocId[i] != aclIdByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid AclIdByDocId cache at " + i); log.warn(".. found .. " + aclIdByDocId[i]); log.warn(".. expected .. " + checkAclIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } thisTestOk = true; for (int i = 0; i < checkTxIdByDocId.length; i++) { if (checkTxIdByDocId[i] != txByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid txByDocId cache at " + i); log.warn(".. found .. " + txByDocId[i]); log.warn(".. expected .. " + checkTxIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } thisTestOk = true; for (int i = 0; i < checkAclTxIdByDocId.length; i++) { if (checkAclTxIdByDocId[i] != aclTxByDocId[i]) { if (thisTestOk) { log.warn("Core " + newSearcher.getIndexDir()); log.warn("Invalid aclTxByDocId cache at " + i); log.warn(".. found .. " + aclTxByDocId[i]); log.warn(".. expected .. " + checkAclTxIdByDocId[i]); try { log.warn(".. expected .. " + newSearcher.doc(i)); } catch (IOException e) { log.error("IO Exception", e); } ok = false; thisTestOk = false; } } } if (!ok) { indexedByDocId.copyFrom(checkIndexedByDocId); allLeafDocs = checkAllLeafDocs; aclIdByDocId = checkAclIdByDocId; txByDocId = checkTxIdByDocId; aclTxByDocId = checkAclTxIdByDocId; log.warn("... Using recomputed cache"); } else { log.info("... cache OK"); } } } catch (IllegalStateException ise) { log.info("Cache state error -> rebuilding", ise); buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); } } } else { buildCacheForReader(indexedByDocId, allLeafDocs, aclIdByDocId, txByDocId, aclTxByDocId, newReader, 0, newReader.maxDoc(), new HashMap<Long, CacheEntry>(), ownerIdManager); } long endTime = System.nanoTime(); log.info("Core cache rebuilt in " + ((endTime - startTime) / (1.0e9))); startTime = System.nanoTime(); int size = doPermissionChecks ? (int) allLeafDocs.cardinality() : 0; ResizeableArrayList<CacheEntry> indexedOderedByAclIdThenDoc = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_ACL_ID_THEN_LEAF); indexedOderedByAclIdThenDoc.resize(size); ResizeableArrayList<CacheEntry> indexedOderedByOwnerIdThenDoc = (ResizeableArrayList<CacheEntry>) newSearcher .cacheLookup(ALFRESCO_ARRAYLIST_CACHE, KEY_DBID_LEAF_PATH_BY_OWNER_ID_THEN_LEAF); indexedOderedByOwnerIdThenDoc.resize(size); if (doPermissionChecks) { int doc = -1; int pos = 0; while ((doc = allLeafDocs.nextSetBit(doc + 1)) != -1) { CacheEntry entry = indexedByDocId.get(doc); indexedOderedByAclIdThenDoc.set(pos, entry); indexedOderedByOwnerIdThenDoc.set(pos, entry); pos++; } } indexedOderedByAclIdThenDoc.sort(new Comparator<CacheEntry>() { @Override public int compare(CacheEntry o1, CacheEntry o2) { if (o2 == null) { if (o1 == null) { return 0; } else { return -1; // nulls at the end } } else { if (o1 == null) { return 1; } else { long diff = o1.getAclid() - o2.getAclid(); if (diff == 0L) { return o1.getLeaf() - o2.getLeaf(); } else { return (diff > 0L) ? 1 : -1; } } } } }); // build lookups HashMap<AclLookUp, AclLookUp> aclLookUp = new HashMap<AclLookUp, AclLookUp>(); AclLookUp currentAclLookUp = null; for (int i = 0; i < indexedOderedByAclIdThenDoc.size(); i++) { CacheEntry entry = indexedOderedByAclIdThenDoc.get(i); if (entry != null) { if (currentAclLookUp == null) { currentAclLookUp = new AclLookUp(entry.getAclid(), i); } else { if (currentAclLookUp.aclid == entry.aclid) { // carry on } else { // acl id has changed - new set currentAclLookUp.setEnd(i); AclLookUp next = new AclLookUp(entry.getAclid(), i); aclLookUp.put(currentAclLookUp, currentAclLookUp); currentAclLookUp = next; } } } else { // found first null we are done if (currentAclLookUp != null) { currentAclLookUp.setEnd(i); aclLookUp.put(currentAclLookUp, currentAclLookUp); } break; } } if (currentAclLookUp != null) { currentAclLookUp.setEnd(indexedOderedByAclIdThenDoc.size()); aclLookUp.put(currentAclLookUp, currentAclLookUp); } indexedOderedByOwnerIdThenDoc.sort(new Comparator<CacheEntry>() { @Override public int compare(CacheEntry o1, CacheEntry o2) { if (o2 == null) { if (o1 == null) { return 0; } else { return -1; // nulls at the end } } else { if (o1 == null) { return 1; } else { int diff = o1.getOwner() - o2.getOwner(); if (diff == 0) { return o1.getLeaf() - o2.getLeaf(); } else { return diff; } } } } }); // build lookups HashMap<String, OwnerLookUp> ownerLookUp = new HashMap<String, OwnerLookUp>(); OwnerLookUp currentOwnerLookUp = null; for (int i = 0; i < indexedOderedByOwnerIdThenDoc.size(); i++) { CacheEntry entry = indexedOderedByOwnerIdThenDoc.get(i); if (entry != null) { if (currentOwnerLookUp == null) { currentOwnerLookUp = new OwnerLookUp(entry.getOwner(), i); } else { if (currentOwnerLookUp.owner == entry.owner) { // carry on } else { // acl id has changed - new set currentOwnerLookUp.setEnd(i); OwnerLookUp next = new OwnerLookUp(entry.getOwner(), i); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } currentOwnerLookUp = next; } } } else { // found first null we are done if (currentOwnerLookUp != null) { currentOwnerLookUp.setEnd(i); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } } break; } } if (currentOwnerLookUp != null) { currentOwnerLookUp.setEnd(indexedOderedByOwnerIdThenDoc.size()); try { ownerLookUp.put(ownerIdManager.get(currentOwnerLookUp.owner), currentOwnerLookUp); } catch (IndexOutOfBoundsException e) { log.warn(" " + ownerIdManager); log.warn(" looking for " + currentOwnerLookUp.owner); throw e; } } // cache readers and acl doc ids //HashMap<String, HashSet<Long>> readerToAclIds = new HashMap<String, HashSet<Long>>(); BitDocSet publicDocSet = new BitDocSet(new OpenBitSet(newReader.maxDoc())); if (doPermissionChecks) { try { HashSet<Long> globallyReadableAcls = buildReaderAclIds(newSearcher, "GROUP_EVERYONE", aclIdByDocId); newSearcher.cacheInsert(AlfrescoSolrEventListener.ALFRESCO_READER_TO_ACL_IDS_CACHE, "GROUP_EVERYONE", globallyReadableAcls); AclLookUp key = new AclLookUp(0); for (Long longAcl : globallyReadableAcls) { key.setAclid(longAcl); AlfrescoSolrEventListener.AclLookUp value = aclLookUp.get(key); if (value != null) { for (int i = value.getStart(); i < value.getEnd(); i++) { publicDocSet.add(indexedOderedByAclIdThenDoc.get(i).getLeaf()); } } } } catch (IOException e) { log.error("IO Exception while warming searcher", e); } } // transform to readers to acl ids endTime = System.nanoTime(); log.info("Derived caches rebuilt in " + ((endTime - startTime) / (1.0e9))); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_ID_BY_DOC_ID, aclIdByDocId); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_TX_ID_BY_DOC_ID, txByDocId); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_TX_ID_BY_DOC_ID, aclTxByDocId); // TODO: Make global readers configurable. globalReaders.add(PermissionService.OWNER_AUTHORITY); globalReaders.add(PermissionService.ADMINISTRATOR_AUTHORITY); globalReaders.add(AuthenticationUtil.getSystemUserName()); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_GLOBAL_READERS, globalReaders); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ALL_LEAF_DOCS, allLeafDocs); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_ACL_LOOKUP, aclLookUp); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_OWNER_LOOKUP, ownerLookUp); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_OWNER_ID_MANAGER, ownerIdManager); newSearcher.cacheInsert(ALFRESCO_CACHE, KEY_PUBLIC_DOC_SET, publicDocSet); try { if (currentSearcher != null) { newSearcher.warm(currentSearcher); } } catch (IOException e) { log.error("IO Exception while warming searcher", e); } }
From source file:org.alfresco.solr.FilterCacheRegenerator.java
License:Open Source License
@Override public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, SolrCache oldCache, Object oldKey, Object oldVal) throws IOException { if (oldKey instanceof Query) { Object cache = newSearcher.cacheLookup(AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_ALL_LEAF_DOCS); if (cache != null) { newSearcher.cacheDocSet((Query) oldKey, null, false); }/* w ww . j av a 2 s . c o m*/ } return true; }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
@Override public Iterable<Map.Entry<String, Object>> getCoreStats() throws IOException { DecimalFormat df = new DecimalFormat("###,###.######"); NamedList<Object> coreSummary = new SimpleOrderedMap<Object>(); RefCounted<SolrIndexSearcher> refCounted = null; try {// w w w .j a v a 2s .c om refCounted = core.getSearcher(false, true, null); SolrIndexSearcher solrIndexSearcher = refCounted.get(); OpenBitSet allLeafDocs = (OpenBitSet) solrIndexSearcher.cacheLookup( AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_ALL_LEAF_DOCS); long count = allLeafDocs.cardinality(); coreSummary.add("Alfresco Nodes in Index", count); coreSummary.add("Searcher", solrIndexSearcher.getStatistics()); Map<String, SolrInfoMBean> infoRegistry = core.getInfoRegistry(); for (String key : infoRegistry.keySet()) { SolrInfoMBean infoMBean = infoRegistry.get(key); if (key.equals("/alfresco")) { coreSummary.add("/alfresco", fixStats(infoMBean.getStatistics())); } if (key.equals("/afts")) { coreSummary.add("/afts", fixStats(infoMBean.getStatistics())); } if (key.equals("/cmis")) { coreSummary.add("/cmis", fixStats(infoMBean.getStatistics())); } if (key.equals("filterCache")) { coreSummary.add("/filterCache", infoMBean.getStatistics()); } if (key.equals("queryResultCache")) { coreSummary.add("/queryResultCache", infoMBean.getStatistics()); } if (key.equals("alfrescoAuthorityCache")) { coreSummary.add("/alfrescoAuthorityCache", infoMBean.getStatistics()); } if (key.equals("alfrescoPathCache")) { coreSummary.add("/alfrescoPathCache", infoMBean.getStatistics()); } } // Find searchers and do memory use for each .. and add them all up7 long memory = 0L; int searcherCount = 0; List<SolrIndexSearcher> searchers = getRegisteredSearchers(); for (SolrIndexSearcher searcher : searchers) { memory += addSearcherStats(coreSummary, searcher, searcherCount); searcherCount++; } coreSummary.add("Number of Searchers", searchers.size()); coreSummary.add("Total Searcher Cache (GB)", df.format(memory / 1024.0f / 1024.0f / 1024.0f)); IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); IndexCommit indexCommit = delPolicy.getLatestCommit(); // race? if (indexCommit == null) { indexCommit = solrIndexSearcher.getReader().getIndexCommit(); } if (indexCommit != null) { delPolicy.setReserveDuration(indexCommit.getVersion(), 20000); Long fileSize = 0L; File dir = new File(solrIndexSearcher.getIndexDir()); for (String name : (Collection<String>) indexCommit.getFileNames()) { File file = new File(dir, name); if (file.exists()) { fileSize += file.length(); } } coreSummary.add("On disk (GB)", df.format(fileSize / 1024.0f / 1024.0f / 1024.0f)); coreSummary.add("Per node B", count > 0 ? fileSize / count : 0); } } finally { if (refCounted != null) { refCounted.decref(); } } return coreSummary; }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
private void doUpdateDescendantAuxDocs(NodeMetaData parentNodeMetaData, boolean overwrite, SolrIndexSearcher solrIndexSearcher, LinkedHashSet<Long> stack, DocSet skippingDocs) throws AuthenticationException, IOException, JSONException { if ((skipDescendantAuxDocsForSpecificTypes && typesForSkippingDescendantAuxDocs.contains(parentNodeMetaData.getType())) || (skipDescendantAuxDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects()))) { return;/*w ww. ja v a2 s . co m*/ } HashSet<Long> childIds = new HashSet<Long>(); if (parentNodeMetaData.getChildIds() != null) { childIds.addAll(parentNodeMetaData.getChildIds()); } BooleanQuery bQuery = new BooleanQuery(); bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_PARENT, parentNodeMetaData.getNodeRef().toString())), Occur.MUST); DocSet docSet = solrIndexSearcher.getDocSet(bQuery); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) solrIndexSearcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID); if (docSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) docSet; OpenBitSet openBitSet = source.getBits(); int current = -1; while ((current = openBitSet.nextSetBit(current + 1)) != -1) { if (!skippingDocs.exists(current)) { CacheEntry entry = indexedByDocId.get(current); childIds.add(entry.getDbid()); } } } else { for (DocIterator it = docSet.iterator(); it.hasNext(); /* */) { int current = it.nextDoc(); if (!skippingDocs.exists(current)) { CacheEntry entry = indexedByDocId.get(current); childIds.add(entry.getDbid()); } } } for (Long childId : childIds) { if (!shouldElementBeIgnored(childId, solrIndexSearcher, skippingDocs)) { NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(childId); nmdp.setToNodeId(childId); nmdp.setIncludeAclId(true); nmdp.setIncludeAspects(true); nmdp.setIncludeChildAssociations(false); nmdp.setIncludeChildIds(true); nmdp.setIncludeNodeRef(true); nmdp.setIncludeOwner(true); nmdp.setIncludeParentAssociations(true); nmdp.setIncludePaths(true); nmdp.setIncludeProperties(false); nmdp.setIncludeType(true); nmdp.setIncludeTxnId(true); // call back to core tracker to talk to client List<NodeMetaData> nodeMetaDatas = coreTracker.getNodesMetaData(nmdp, MAX_RESULTS_NODES_META_DATA); for (NodeMetaData nodeMetaData : nodeMetaDatas) { if (mayHaveChildren(nodeMetaData)) { updateDescendantAuxDocs(nodeMetaData, overwrite, solrIndexSearcher, stack, skippingDocs); } // Avoid adding aux docs for stuff yet to be indexed or unindexed (via the index control aspect) log.info(".. checking aux doc exists in index before we update it"); Query query = new TermQuery(new Term(QueryConstants.FIELD_ID, "AUX-" + childId)); DocSet auxSet = solrIndexSearcher.getDocSet(query); if (auxSet.size() > 0) { log.debug("... cascade update aux doc " + childId); SolrInputDocument aux = createAuxDoc(nodeMetaData); AddUpdateCommand auxDocCmd = new AddUpdateCommand(); auxDocCmd.overwriteCommitted = overwrite; auxDocCmd.overwritePending = overwrite; auxDocCmd.solrDoc = aux; auxDocCmd.doc = toDocument(auxDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); core.getUpdateHandler().addDoc(auxDocCmd); } else { log.debug("... no aux doc found to update " + childId); } } } } }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
private long addSearcherStats(NamedList<Object> coreSummary, SolrIndexSearcher solrIndexSearcher, int index) { DecimalFormat df = new DecimalFormat("###,###.######"); OpenBitSet allLeafDocs = (OpenBitSet) solrIndexSearcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_ALL_LEAF_DOCS); long count = allLeafDocs.cardinality(); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) solrIndexSearcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID); ResizeableArrayList<CacheEntry> indexedOderedByAclIdThenDoc = (ResizeableArrayList<CacheEntry>) solrIndexSearcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_ACL_ID_THEN_LEAF); long memory = (count * 40) + (indexedByDocId.size() * 8 * 4) + (indexedOderedByAclIdThenDoc.size() * 8 * 2); memory += (filterCacheSize + pathCacheSize + authorityCacheSize) * indexedByDocId.size() / 8; NamedList<Object> details = new SimpleOrderedMap<Object>(); details.add("Searcher", solrIndexSearcher.getStatistics()); details.add("Size", indexedByDocId.size()); details.add("Node Count", count); details.add("Memory (GB)", df.format(memory / 1024.0f / 1024.0f / 1024.0f)); coreSummary.add("Searcher-" + index, details); return memory; }
From source file:org.alfresco.solr.query.SolrAuthorityScorer.java
License:Open Source License
public static SolrAuthorityScorer createAuthorityScorer(Weight weight, LeafReaderContext context, SolrIndexSearcher searcher, String authority) throws IOException { Properties p = searcher.getSchema().getResourceLoader().getCoreProperties(); boolean doPermissionChecks = Boolean.parseBoolean(p.getProperty("alfresco.doPermissionChecks", "true")); Query key = new SolrAuthorityQuery(authority); DocSet answer = (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key); if (answer != null) { // Answer was in the cache, so return it. return new SolrAuthorityScorer(weight, answer, context, searcher); }//from w w w . java 2s . c o m // Answer was not in cache, so build the results, cache and return. final HashSet<String> globalReaders = GlobalReaders.getReaders(); if (globalReaders.contains(authority) || (doPermissionChecks == false)) { // can read all DocSet allDocs = searcher.getDocSet(new MatchAllDocsQuery()); return new SolrAuthorityScorer(weight, allDocs, context, searcher); } // Docs for which the authority has explicit read access. DocSet readableDocSet = searcher.getDocSet(new SolrReaderQuery(authority)); // Are all doc owners granted read permissions at a global level? if (globalReaders.contains(PermissionService.OWNER_AUTHORITY)) { // Get the set of docs owned by the authority (which they can therefore read). DocSet authorityOwnedDocs = searcher.getDocSet(new SolrOwnerQuery(authority)); // Final set of docs that the authority can read. DocSet toCache = readableDocSet.union(authorityOwnedDocs); searcher.cacheInsert(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrAuthorityScorer(weight, toCache, context, searcher); } else { // for that docs I own that have owner Read rights DocSet ownerReadableDocSet = searcher.getDocSet(new SolrReaderQuery(PermissionService.OWNER_AUTHORITY)); DocSet authorityOwnedDocs = searcher.getDocSet(new SolrOwnerQuery(authority)); // Docs where the authority is an owner and where owners have read rights. DocSet docsAuthorityOwnsAndCanRead = ownerReadableDocSet.intersection(authorityOwnedDocs); // Final set of docs that the authority can read. DocSet toCache = readableDocSet.union(docsAuthorityOwnsAndCanRead); searcher.cacheInsert(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrAuthorityScorer(weight, toCache, context, searcher); } }
From source file:org.alfresco.solr.query.SolrAuthoritySetScorer.java
License:Open Source License
public static SolrAuthoritySetScorer createAuthoritySetScorer(Weight weight, LeafReaderContext context, SolrIndexSearcher searcher, String authorities) throws IOException { Properties p = searcher.getSchema().getResourceLoader().getCoreProperties(); boolean doPermissionChecks = Boolean.parseBoolean(p.getProperty("alfresco.doPermissionChecks", "true")); Query key = new SolrAuthoritySetQuery(authorities); DocSet answer = (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key); if (answer != null) { // Answer was in the cache, so return it. return new SolrAuthoritySetScorer(weight, answer, context, searcher); }//from ww w. jav a2s . c o m // Answer was not in cache, so build the results, cache and return. String[] auths = authorities.substring(1).split(authorities.substring(0, 1)); boolean hasGlobalRead = false; final HashSet<String> globalReaders = GlobalReaders.getReaders(); for (String auth : auths) { if (globalReaders.contains(auth)) { hasGlobalRead = true; break; } } if (hasGlobalRead || (doPermissionChecks == false)) { // can read all WrappedQuery wrapped = new WrappedQuery(new MatchAllDocsQuery()); wrapped.setCache(false); DocSet allDocs = searcher.getDocSet(wrapped); return new SolrAuthoritySetScorer(weight, allDocs, context, searcher); } // Docs for which the authorities have explicit read access. WrappedQuery wrapped; wrapped = new WrappedQuery(new SolrReaderSetQuery(authorities)); wrapped.setCache(false); DocSet readableDocSet = searcher.getDocSet(wrapped); // Are all doc owners granted read permissions at a global level? if (globalReaders.contains(PermissionService.OWNER_AUTHORITY)) { // Get the set of docs owned by the authorities (which they can therefore read). wrapped = new WrappedQuery(new SolrOwnerSetQuery(authorities)); wrapped.setCache(false); DocSet authorityOwnedDocs = searcher.getDocSet(wrapped); // Final set of docs that the authorities can read. DocSet toCache = readableDocSet.union(authorityOwnedDocs); searcher.cacheInsert(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrAuthoritySetScorer(weight, toCache, context, searcher); } else { // for that docs I own that have owner Read rights wrapped = new WrappedQuery(new SolrReaderSetQuery("|" + PermissionService.OWNER_AUTHORITY)); wrapped.setCache(false); DocSet ownerReadableDocSet = searcher.getDocSet(wrapped); wrapped = new WrappedQuery(new SolrOwnerSetQuery(authorities)); wrapped.setCache(false); DocSet authorityOwnedDocs = searcher.getDocSet(wrapped); // Docs where the authority is an owner and where owners have read rights. DocSet docsAuthorityOwnsAndCanRead = ownerReadableDocSet.intersection(authorityOwnedDocs); // Final set of docs that the authorities can read. DocSet toCache = readableDocSet.union(docsAuthorityOwnsAndCanRead); searcher.cacheInsert(CacheConstants.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrAuthoritySetScorer(weight, toCache, context, searcher); } }
From source file:org.alfresco.solr.query.SolrCachingAuthorityScorer.java
License:Open Source License
public static SolrCachingAuthorityScorer createAuthorityScorer(SolrIndexSearcher searcher, Similarity similarity, String authority, SolrIndexReader reader) throws IOException { // Get hold of solr top level searcher // Execute query with caching // translate reults to leaf docs // build ordered doc list Query key = new SolrCachingAuthorityQuery(authority); DocSet answer = (DocSet) searcher.cacheLookup(AlfrescoSolrEventListener.ALFRESCO_AUTHORITY_CACHE, key); if (answer != null) { return new SolrCachingAuthorityScorer(similarity, answer, reader); }/*from ww w .j ava 2 s . c o m*/ HashSet<String> globalReaders = (HashSet<String>) searcher.cacheLookup( AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_GLOBAL_READERS); if (globalReaders.contains(authority)) { // can read all OpenBitSet allLeafDocs = (OpenBitSet) searcher.cacheLookup(AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_ALL_LEAF_DOCS); DocSet toCache = new BitDocSet(allLeafDocs); searcher.cacheInsert(AlfrescoSolrEventListener.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrCachingAuthorityScorer(similarity, toCache, reader); } DocSet readableDocSet = searcher.getDocSet(new SolrCachingReaderQuery(authority)); if (globalReaders.contains(PermissionService.OWNER_AUTHORITY)) { DocSet authorityOwnedDocs = searcher.getDocSet(new SolrCachingOwnerQuery(authority)); DocSet toCache = readableDocSet.union(authorityOwnedDocs); searcher.cacheInsert(AlfrescoSolrEventListener.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrCachingAuthorityScorer(similarity, toCache, reader); } else { // for that docs I own that have owner Read rights DocSet ownerReadableDocSet = searcher .getDocSet(new SolrCachingReaderQuery(PermissionService.OWNER_AUTHORITY)); DocSet authorityOwnedDocs = searcher.getDocSet(new SolrCachingOwnerQuery(authority)); DocSet docsAuthorityOwnsAndCanRead = ownerReadableDocSet.intersection(authorityOwnedDocs); DocSet toCache = readableDocSet.union(docsAuthorityOwnsAndCanRead); searcher.cacheInsert(AlfrescoSolrEventListener.ALFRESCO_AUTHORITY_CACHE, key, toCache); return new SolrCachingAuthorityScorer(similarity, toCache, reader); } }
From source file:org.alfresco.solr.query.SolrCachingAuxDocScorer.java
License:Open Source License
public static SolrCachingAuxDocScorer createAuxDocScorer(SolrIndexSearcher searcher, Similarity similarity, Query query, SolrIndexReader reader) throws IOException { // Get hold of solr top level searcher // Execute query with caching // translate reults to leaf docs // build ordered doc list DocSet auxDocSet = searcher.getDocSet(new ContextAwareQuery(query, null)); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) searcher.cacheLookup( AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID); // List<ScoreDoc> auxDocs = pathCollector.getDocs(); OpenBitSet translated = new OpenBitSet(searcher.getReader().maxDoc()); if (auxDocSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) auxDocSet; OpenBitSet openBitSet = source.getBits(); int current = -1; while ((current = openBitSet.nextSetBit(current + 1)) != -1) { CacheEntry entry = indexedByDocId.get(current); translated.set(entry.getLeaf()); }/*from ww w. j av a2 s. c om*/ } else { for (DocIterator it = auxDocSet.iterator(); it.hasNext(); /* */) { CacheEntry entry = indexedByDocId.get(it.nextDoc()); translated.set(entry.getLeaf()); } } return new SolrCachingAuxDocScorer(similarity, new BitDocSet(translated), reader); }
From source file:org.alfresco.solr.query.SolrCachingOwnerScorer.java
License:Open Source License
public static SolrCachingOwnerScorer createOwnerScorer(SolrIndexSearcher searcher, Similarity similarity, String authority, SolrIndexReader reader) throws IOException { // Get hold of solr top level searcher // Execute query with caching // translate reults to leaf docs // build ordered doc list BitDocSet authorityOwnedDocs = new BitDocSet(new OpenBitSet(searcher.getReader().maxDoc())); HashMap<String, OwnerLookUp> ownerLookUp = (HashMap<String, OwnerLookUp>) searcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_OWNER_LOOKUP); if (authority.contains("\\|")) { for (String current : authority.split("|")) { OwnerLookUp lookUp = ownerLookUp.get(current); if (lookUp != null) { ResizeableArrayList<CacheEntry> indexedOderedByOwnerIdThenDoc = (ResizeableArrayList<CacheEntry>) searcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_OWNER_ID_THEN_LEAF); for (int i = lookUp.getStart(); i < lookUp.getEnd(); i++) { authorityOwnedDocs.addUnique(indexedOderedByOwnerIdThenDoc.get(i).getLeaf()); }/*from w w w .j av a 2 s . c o m*/ } } } else { OwnerLookUp lookUp = ownerLookUp.get(authority); if (lookUp != null) { ResizeableArrayList<CacheEntry> indexedOderedByOwnerIdThenDoc = (ResizeableArrayList<CacheEntry>) searcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_OWNER_ID_THEN_LEAF); for (int i = lookUp.getStart(); i < lookUp.getEnd(); i++) { authorityOwnedDocs.addUnique(indexedOderedByOwnerIdThenDoc.get(i).getLeaf()); } } } return new SolrCachingOwnerScorer(similarity, authorityOwnedDocs, reader); }