List of usage examples for org.apache.solr.search DelegatingCollector setLastDelegate
public void setLastDelegate(Collector delegate)
From source file:org.alfresco.solr.SolrInformationServer.java
License:Open Source License
@Override public List<TenantAclIdDbId> getDocsWithUncleanContent(int start, int rows) throws IOException { RefCounted<SolrIndexSearcher> refCounted = null; try {//from ww w. j av a2 s.com List<TenantAclIdDbId> docIds = new ArrayList<>(); refCounted = this.core.getSearcher(); SolrIndexSearcher searcher = refCounted.get(); /* * Below is the code for purging the cleanContentCache. * The cleanContentCache is an in-memory LRU cache of the transactions that have already * had their content fetched. This is needed because the ContentTracker does not have an up-to-date * snapshot of the index to determine which nodes are marked as dirty/new. The cleanContentCache is used * to filter out nodes that belong to transactions that have already been processed, which stops them from * being re-processed. * * The cleanContentCache needs to be purged periodically to support retrying of failed content fetches. * This is because fetches for individual nodes within the transaction may have failed, but the transaction will still be in the * cleanContentCache, which prevents it from being retried. * * Once a transaction is purged from the cleanContentCache it will be retried automatically if it is marked dirty/new * in current snapshot of the index. * * The code below runs every two minutes and purges transactions from the * cleanContentCache that is more then 20 minutes old. * */ long purgeTime = System.currentTimeMillis(); if (purgeTime - cleanContentLastPurged > 120000) { Iterator<Entry<Long, Long>> entries = cleanContentCache.entrySet().iterator(); while (entries.hasNext()) { Entry<Long, Long> entry = entries.next(); long txnTime = entry.getValue(); if (purgeTime - txnTime > 1200000) { //Purge the clean content cache of records more then 20 minutes old. entries.remove(); } } cleanContentLastPurged = purgeTime; } long txnFloor; Sort sort = new Sort(new SortField(FIELD_INTXID, SortField.Type.LONG)); sort = sort.rewrite(searcher); TopFieldCollector collector = TopFieldCollector.create(sort, 1, null, false, false, false); DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanContentCache); //Filter transactions that have already been processed. delegatingCollector.setLastDelegate(collector); searcher.search(dirtyOrNewContentQuery(), delegatingCollector); if (collector.getTotalHits() == 0) { return docIds; } ScoreDoc[] scoreDocs = collector.topDocs().scoreDocs; List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves(); int index = ReaderUtil.subIndex(scoreDocs[0].doc, leaves); LeafReaderContext context = leaves.get(index); NumericDocValues longs = context.reader().getNumericDocValues(FIELD_INTXID); txnFloor = longs.get(scoreDocs[0].doc - context.docBase); //Find the next N transactions //The TxnCollector collects the transaction ids from the matching documents //The txnIds are limited to a range >= the txnFloor and < an arbitrary transaction ceiling. TxnCollector txnCollector = new TxnCollector(txnFloor); searcher.search(dirtyOrNewContentQuery(), txnCollector); LongHashSet txnSet = txnCollector.getTxnSet(); if (txnSet.size() == 0) { //This should really never be the case, at a minimum the transaction floor should be collected. return docIds; } FieldType fieldType = searcher.getSchema().getField(FIELD_INTXID).getType(); BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (LongCursor cursor : txnSet) { long txnID = cursor.value; //Build up the query for the filter of transactions we need to pull the dirty content for. TermQuery txnIDQuery = new TermQuery( new Term(FIELD_INTXID, fieldType.readableToIndexed(Long.toString(txnID)))); builder.add(new BooleanClause(txnIDQuery, BooleanClause.Occur.SHOULD)); } BooleanQuery txnFilterQuery = builder.build(); //Get the docs with dirty content for the transactions gathered above. DocListCollector docListCollector = new DocListCollector(); BooleanQuery.Builder builder2 = new BooleanQuery.Builder(); builder2.add(dirtyOrNewContentQuery(), BooleanClause.Occur.MUST); builder2.add(new QueryWrapperFilter(txnFilterQuery), BooleanClause.Occur.MUST); searcher.search(builder2.build(), docListCollector); IntArrayList docList = docListCollector.getDocs(); int size = docList.size(); List<Long> processedTxns = new ArrayList<>(); for (int i = 0; i < size; ++i) { int doc = docList.get(i); Document document = searcher.doc(doc, REQUEST_ONLY_ID_FIELD); index = ReaderUtil.subIndex(doc, leaves); context = leaves.get(index); longs = context.reader().getNumericDocValues(FIELD_INTXID); long txnId = longs.get(doc - context.docBase); if (!cleanContentCache.containsKey(txnId)) { processedTxns.add(txnId); IndexableField id = document.getField(FIELD_SOLR4_ID); String idString = id.stringValue(); TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(idString); docIds.add(tenantAndDbId); } } long txnTime = System.currentTimeMillis(); for (Long l : processedTxns) { //Save the indexVersion so we know when we can clean out this entry cleanContentCache.put(l, txnTime); } return docIds; } finally { ofNullable(refCounted).ifPresent(RefCounted::decref); } }
From source file:org.alfresco.solr.SolrInformationServer.java
License:Open Source License
@Override public List<Transaction> getCascades(int num) throws IOException { RefCounted<SolrIndexSearcher> refCounted = null; try {//from w ww . j a v a 2s .co m refCounted = this.core.getSearcher(); SolrIndexSearcher searcher = refCounted.get(); Collector collector; TopFieldCollector topFieldCollector = TopFieldCollector.create( new Sort(new SortField(FIELD_TXID, SortField.Type.LONG)), num, null, false, false, false); collector = topFieldCollector; LegacyNumericRangeQuery q = LegacyNumericRangeQuery.newIntRange(FIELD_CASCADE_FLAG, 1, 1, true, true); DelegatingCollector delegatingCollector = new TxnCacheFilter(cleanCascadeCache); delegatingCollector.setLastDelegate(collector); collector = delegatingCollector; searcher.search(q, collector); ScoreDoc[] scoreDocs = topFieldCollector.topDocs().scoreDocs; Set<String> fields = new HashSet<>(); fields.add(FIELD_S_TXID); fields.add(FIELD_S_TXCOMMITTIME); List<Transaction> transactions = new ArrayList<>(scoreDocs.length); for (ScoreDoc scoreDoc : scoreDocs) { Transaction transaction = new Transaction(); Document doc = searcher.doc(scoreDoc.doc, fields); IndexableField txID = doc.getField(FIELD_S_TXID); long txnID = txID.numericValue().longValue(); cleanCascadeCache.put(txnID, null); transaction.setId(txnID); IndexableField txnCommitTime = doc.getField(FIELD_S_TXCOMMITTIME); transaction.setCommitTimeMs(txnCommitTime.numericValue().longValue()); transactions.add(transaction); } return transactions; } finally { ofNullable(refCounted).ifPresent(RefCounted::decref); } }