List of usage examples for org.apache.solr.search SolrIndexSearcher getDocSet
public DocSet getDocSet(DocsEnumState deState) throws IOException
From source file:com.doculibre.constellio.solr.handler.component.ConstellioAuthorizationComponent.java
License:Open Source License
@SuppressWarnings("unchecked") @Override/*from w w w. j a v a 2s .c o m*/ public void prepare(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrIndexSearcher searcher = req.getSearcher(); //IndexReader reader = req.getSearcher().getReader(); SolrParams params = req.getParams(); // A runtime param can skip if (!params.getBool(ENABLE, true)) { return; } Query query = rb.getQuery(); String qstr = rb.getQueryString(); if (query == null || qstr == null) { return; } ConstellioUser user; String userIdStr = params.get(ConstellioSolrQueryParams.USER_ID); if (userIdStr != null) { UserServices userServices = ConstellioSpringUtils.getUserServices(); try { user = userServices.get(new Long(userIdStr)); } catch (NumberFormatException e) { user = null; } } else { user = null; } String collectionName = params.get(ConstellioSolrQueryParams.COLLECTION_NAME); RecordCollectionServices collectionServices = ConstellioSpringUtils.getRecordCollectionServices(); FederationServices federationServices = ConstellioSpringUtils.getFederationServices(); RecordCollection collection = collectionServices.get(collectionName); List<TermQuery> restrictedCollectionQueries = new ArrayList<TermQuery>(); if (collection.isFederationOwner()) { List<RecordCollection> includedCollections = federationServices.listIncludedCollections(collection); for (RecordCollection includedCollection : includedCollections) { if (includedCollection.hasSearchPermission() && (user == null || !user.hasSearchPermission(includedCollection))) { restrictedCollectionQueries.add(new TermQuery( new Term(IndexField.COLLECTION_ID_FIELD, "" + includedCollection.getId()))); } } } // User must be logged in to see private records if (user != null) { String luceneQueryStr = params.get(ConstellioSolrQueryParams.LUCENE_QUERY); if (StringUtils.isBlank(luceneQueryStr)) { return; } IndexSchema schema = req.getSchema(); SolrQueryParser queryParser = new SolrQueryParser(rb.getQparser(), IndexField.DEFAULT_SEARCH_FIELD); Query luceneQuery; try { luceneQuery = queryParser.parse(luceneQueryStr); } catch (SyntaxError e) { log.error("Error parsing lucene query " + luceneQueryStr, e); return; } // Create a new query which will only include private records BooleanQuery privateRecordQuery = new BooleanQuery(true); privateRecordQuery.add(luceneQuery, BooleanClause.Occur.MUST); for (TermQuery restrictionCollectionQuery : restrictedCollectionQueries) { privateRecordQuery.add(restrictionCollectionQuery, BooleanClause.Occur.MUST_NOT); } TermQuery privateRecordTQ = new TermQuery(new Term(IndexField.PUBLIC_RECORD_FIELD, "F")); privateRecordQuery.add(privateRecordTQ, BooleanClause.Occur.MUST); DocSet privateRecordIdDocSet = searcher.getDocSet(privateRecordQuery); if (privateRecordIdDocSet.size() > 0) { RecordServices recordServices = ConstellioSpringUtils.getRecordServices(); ACLServices aclServices = ConstellioSpringUtils.getACLServices(); ConnectorManagerServices connectorManagerServices = ConstellioSpringUtils .getConnectorManagerServices(); List<Record> privateRecords = new ArrayList<Record>(); DocIterator docIt = privateRecordIdDocSet.iterator(); while (docIt.hasNext()) { int docId = docIt.nextDoc(); Document luceneDoc = searcher.doc(docId); Long recordId = new Long(luceneDoc.get(IndexField.RECORD_ID_FIELD)); Record record = recordServices.get(recordId, collection); privateRecords.add(record); } // First pass : Remove ACL authorized records List<Record> unevaluatedPrivateRecords = aclServices.removeAuthorizedRecords(privateRecords, user); if (!unevaluatedPrivateRecords.isEmpty()) { Set<UserCredentials> userCredentials = user.getUserCredentials(); // Second pass : Ask the connector manager ConnectorManager connectorManager = connectorManagerServices.getDefaultConnectorManager(); List<Record> authorizedRecords = connectorManagerServices .authorizeByConnector(unevaluatedPrivateRecords, userCredentials, connectorManager); List<Record> unauthorizedRecords = ListUtils.removeAll(unevaluatedPrivateRecords, authorizedRecords); if (!unauthorizedRecords.isEmpty()) { // Create a new query which will exclude unauthorized records BooleanQuery authorizedRecordQuery = new BooleanQuery(true); authorizedRecordQuery.add(query, BooleanClause.Occur.MUST); for (Record unauthorizedRecord : unauthorizedRecords) { TermQuery unauthorizedRecordTQ = new TermQuery( new Term(IndexField.RECORD_ID_FIELD, "" + unauthorizedRecord.getId())); authorizedRecordQuery.add(unauthorizedRecordTQ, BooleanClause.Occur.MUST_NOT); } rb.setQuery(authorizedRecordQuery); } } } } else { BooleanQuery publicRecordQuery = new BooleanQuery(true); publicRecordQuery.add(query, BooleanClause.Occur.MUST); TermQuery publicRecordTQ = new TermQuery(new Term(IndexField.PUBLIC_RECORD_FIELD, "T")); publicRecordQuery.add(publicRecordTQ, BooleanClause.Occur.MUST); for (TermQuery restrictionCollectionQuery : restrictedCollectionQueries) { publicRecordQuery.add(restrictionCollectionQuery, BooleanClause.Occur.MUST_NOT); } rb.setQuery(publicRecordQuery); } }
From source file:com.searchbox.solr.CategoryLikeThis.java
License:Apache License
@Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { numRequests++;/* w ww . ja va 2s.c om*/ long startTime = System.currentTimeMillis(); if (!keystate) { LOGGER.error( "License key failure, not performing clt query. Please email contact@searchbox.com for more information."); return; } try { SolrParams params = req.getParams(); String senseField = params.get(SenseParams.SENSE_FIELD, SenseParams.DEFAULT_SENSE_FIELD); BooleanQuery catfilter = new BooleanQuery(); // Set field flags ReturnFields returnFields = new SolrReturnFields(req); rsp.setReturnFields(returnFields); int flags = 0; if (returnFields.wantsScore()) { flags |= SolrIndexSearcher.GET_SCORES; } String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE); String q = params.get(CommonParams.Q); Query query = null; SortSpec sortSpec = null; List<Query> filters = new LinkedList<Query>(); List<RealTermFreqVector> prototypetfs = new LinkedList<RealTermFreqVector>(); try { if (q != null) { QParser parser = QParser.getParser(q, defType, req); query = parser.getQuery(); sortSpec = parser.getSort(true); } String[] fqs = req.getParams().getParams(CommonParams.FQ); if (fqs != null && fqs.length != 0) { for (String fq : fqs) { if (fq != null && fq.trim().length() != 0) { QParser fqp = QParser.getParser(fq, null, req); filters.add(fqp.getQuery()); } } } } catch (Exception e) { numErrors++; throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); } SolrIndexSearcher searcher = req.getSearcher(); DocListAndSet cltDocs = null; // Parse Required Params // This will either have a single Reader or valid query Reader reader = null; try { if (q == null || q.trim().length() < 1) { Iterable<ContentStream> streams = req.getContentStreams(); if (streams != null) { Iterator<ContentStream> iter = streams.iterator(); if (iter.hasNext()) { reader = iter.next().getReader(); } if (iter.hasNext()) { numErrors++; throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "SenseLikeThis does not support multiple ContentStreams"); } } } int start = params.getInt(CommonParams.START, 0); int rows = params.getInt(CommonParams.ROWS, 10); // Find documents SenseLikeThis - either with a reader or a query // -------------------------------------------------------------------------------- if (reader != null) { numErrors++; throw new RuntimeException("SLT based on a reader is not yet implemented"); } else if (q != null) { LOGGER.debug("Query for category:\t" + query); DocList match = searcher.getDocList(query, null, null, 0, 10, flags); // get first 10 if (match.size() == 0) { // no docs to make prototype! LOGGER.info("No documents found for prototype!"); rsp.add("response", new DocListAndSet()); return; } HashMap<String, Float> overallFreqMap = new HashMap<String, Float>(); // Create the TF of blah blah blah DocIterator iterator = match.iterator(); while (iterator.hasNext()) { // do a MoreLikeThis query for each document in results int id = iterator.nextDoc(); LOGGER.trace("Working on doc:\t" + id); RealTermFreqVector rtv = new RealTermFreqVector(id, searcher.getIndexReader(), senseField); for (int zz = 0; zz < rtv.getSize(); zz++) { Float prev = overallFreqMap.get(rtv.getTerms()[zz]); if (prev == null) { prev = 0f; } overallFreqMap.put(rtv.getTerms()[zz], rtv.getFreqs()[zz] + prev); } prototypetfs.add(rtv); } List<String> sortedKeys = Ordering.natural().onResultOf(Functions.forMap(overallFreqMap)) .immutableSortedCopy(overallFreqMap.keySet()); int keyiter = Math.min(sortedKeys.size() - 1, BooleanQuery.getMaxClauseCount() - 1); LOGGER.debug("I have this many terms:\t" + sortedKeys.size()); LOGGER.debug("And i'm going to use this many:\t" + keyiter); for (; keyiter >= 0; keyiter--) { TermQuery tq = new TermQuery(new Term(senseField, sortedKeys.get(keyiter))); catfilter.add(tq, BooleanClause.Occur.SHOULD); } } else { numErrors++; throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "CategoryLikeThis requires either a query (?q=) or text to find similar documents."); } LOGGER.debug("document filter is: \t" + catfilter); CategorizationBase model = new CategorizationBase(prototypetfs); CategoryQuery clt = CategoryQuery.CategoryQueryForDocument(catfilter, model, searcher.getIndexReader(), senseField); DocSet filtered = searcher.getDocSet(filters); cltDocs = searcher.getDocListAndSet(clt, filtered, Sort.RELEVANCE, start, rows, flags); } finally { if (reader != null) { reader.close(); } } if (cltDocs == null) { numEmpty++; cltDocs = new DocListAndSet(); // avoid NPE } rsp.add("response", cltDocs.docList); // maybe facet the results if (params.getBool(FacetParams.FACET, false)) { if (cltDocs.docSet == null) { rsp.add("facet_counts", null); } else { SimpleFacets f = new SimpleFacets(req, cltDocs.docSet, params); rsp.add("facet_counts", f.getFacetCounts()); } } } catch (Exception e) { numErrors++; } finally { totalTime += System.currentTimeMillis() - startTime; } }
From source file:com.searchbox.SuggesterTreeHolder.java
License:Apache License
SuggestionResultSet getSuggestions(SolrIndexSearcher searcher, String[] fields, String query, int maxPhraseSearch) { query = deAccent(query);//from w ww.j av a 2s . c o m String[] queryTokens = query.replaceAll("[^A-Za-z0-9 ]", " ").replace(" ", " ").trim().split(" "); // TODO should use // tokensizer.. SuggestionResultSet rs = headNode.computeQt(queryTokens[queryTokens.length - 1].toLowerCase(), maxPhraseSearch); // get completion for the first word in the // suggestion // didn't find it, bail early if (rs == null) { return new SuggestionResultSet("", 0); } rs.myval = ""; LOGGER.debug("Doing 2nd part of equation"); try { if (queryTokens.length > 1) { // Solr 4.4 method change QueryParser parser = new QueryParser(Version.LUCENE_44, "contents", searcher.getCore().getLatestSchema().getAnalyzer()); // QueryParser parser = new QueryParser(Version.LUCENE_43, // "contents", searcher.getCore().getSchema().getAnalyzer()); SuggestionResultSet newrs = new SuggestionResultSet("", maxPhraseSearch); StringBuilder sb = new StringBuilder(); // build a search in all of the target fields for (int zz = 0; zz < queryTokens.length - 1; zz++) { newrs.myval = newrs.myval + queryTokens[zz] + " "; StringBuilder inner = new StringBuilder(); for (String field : fields) { String escaped_field = parser.escape(field); // looking for the query token String escaped_token = parser.escape(queryTokens[zz]); inner.append(escaped_field + ":" + escaped_token + " "); } if (inner.length() > 0) { sb.append("+(" + inner + ")"); } } // LOGGER.info("SB query:\t" + sb.toString()); Query q = null; try { // convert it to a solr query q = parser.parse(sb.toString()); // LOGGER.info("BQ1 query:\t" + q.toString()); } catch (Exception e) { e.printStackTrace(); LOGGER.error("Error parsing query:\t" + sb.toString()); } DocSet qd = searcher.getDocSet(q); // LOGGER.info("Number of docs in set\t" + qd.size()); for (SuggestionResult sr : rs.suggestions) { // for each of the possible suggestions, see how prevelant // they are in the document set so that we can know their // likelihood of being correct sb = new StringBuilder(); // should use tokensizer String[] suggestionTokens = sr.suggestion.split(" "); for (int zz = 0; zz < suggestionTokens.length; zz++) { StringBuilder inner = new StringBuilder(); for (String field : fields) { inner.append(field + ":" + suggestionTokens[zz] + " "); } if (inner.length() > 0) { sb.append("+(" + inner + ")"); } } // prevent zero bump down double Q_c = .0000001; try { // LOGGER.info("BQ2 query String:\t" + sb.toString()); q = parser.parse(sb.toString()); // LOGGER.info("BQ2 query query:\t" + q.toString()); } catch (Exception e) { // LOGGER.error("parser fail?"); } DocSet pd = searcher.getDocSet(q); // LOGGER.info("Number of docs in phrase set\t" + // pd.size()); if (pd.size() != 0) { // As per equation (13) from paper Q_c += qd.intersection(pd).size() / (pd.size() * 1.0); } // LOGGER.info("Number of docs in phrase set----- Q_c\t (" + // Q_c + ") * (" + sr.probability + ")"); newrs.add(sr.suggestion, sr.probability * Q_c); } rs = newrs; } } catch (IOException ex) { LOGGER.error(ex.getMessage()); } return rs; }
From source file:org.alfresco.solr.cache.AuthorityCacheRegenerator.java
License:Open Source License
@SuppressWarnings({ "rawtypes" })
@Override/*from www .j a v a 2s. c o m*/
public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, SolrCache oldCache,
Object oldKey, Object oldVal) throws IOException {
if (oldKey instanceof Query) {
// The authority cache contains results keyed by SolrAuthorityQuery
// and SolrAuthoritySetQuery.
Query authQuery = (Query) oldKey;
// Execute the query on the new searcher - resulting in cache population as a side-effect.
newSearcher.getDocSet(authQuery);
}
return true;
}
From source file:org.alfresco.solr.cache.PathCacheRegenerator.java
License:Open Source License
@SuppressWarnings({ "rawtypes" })
@Override/*from w w w. j a va 2 s . c om*/
public boolean regenerateItem(SolrIndexSearcher newSearcher, SolrCache newCache, SolrCache oldCache,
Object oldKey, Object oldVal) throws IOException {
if (oldKey instanceof SolrPathQuery) {
SolrPathQuery pathQuery = (SolrPathQuery) oldKey;
// Re-execute the path query in a cache-aware context - causing new results to be cached.
SolrCachingPathQuery cachingPathQuery = new SolrCachingPathQuery(pathQuery);
newSearcher.getDocSet(cachingPathQuery);
}
return true;
}
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
public NodeReport checkNodeCommon(NodeReport nodeReport) { long dbid = nodeReport.getDbid(); RefCounted<SolrIndexSearcher> refCounted = null; try {/*w w w .ja va2 s . c om*/ refCounted = core.getSearcher(false, true, null); if (refCounted == null) { return nodeReport; } try { SolrIndexSearcher solrIndexSearcher = refCounted.get(); String dbidString = NumericEncoder.encode(dbid); DocSet docSet = solrIndexSearcher.getDocSet(new TermQuery(new Term("DBID", dbidString))); // should find leaf and aux for (DocIterator it = docSet.iterator(); it.hasNext(); /* */) { int doc = it.nextDoc(); Document document = solrIndexSearcher.doc(doc); Fieldable fieldable = document.getFieldable("ID"); if (fieldable != null) { String value = fieldable.stringValue(); if (value != null) { if (value.startsWith("LEAF-")) { nodeReport.setIndexLeafDoc(Long.valueOf(doc)); } else if (value.startsWith("AUX-")) { nodeReport.setIndexAuxDoc(Long.valueOf(doc)); } } } } DocSet txDocSet = solrIndexSearcher.getDocSet(new WildcardQuery(new Term("TXID", "*"))); for (DocIterator it = txDocSet.iterator(); it.hasNext(); /* */) { int doc = it.nextDoc(); Document document = solrIndexSearcher.doc(doc); Fieldable fieldable = document.getFieldable("TXID"); if (fieldable != null) { if ((nodeReport.getIndexLeafDoc() == null) || (doc < nodeReport.getIndexLeafDoc().longValue())) { String value = fieldable.stringValue(); long txid = Long.parseLong(value); nodeReport.setIndexLeafTx(txid); } if ((nodeReport.getIndexAuxDoc() == null) || (doc < nodeReport.getIndexAuxDoc().longValue())) { String value = fieldable.stringValue(); long txid = Long.parseLong(value); nodeReport.setIndexAuxTx(txid); } } } } finally { refCounted.decref(); } } catch (IOException e) { // TODO: do what here? } return nodeReport; }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
@Override public void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException { RefCounted<SolrIndexSearcher> refCounted = null; try {//from ww w. j a v a 2s .co m long start = System.nanoTime(); refCounted = core.getSearcher(false, true, null); SolrIndexSearcher solrIndexSearcher = refCounted.get(); if ((node.getStatus() == SolrApiNodeStatus.DELETED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) { // fix up any secondary paths NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(node.getId()); nmdp.setToNodeId(node.getId()); List<NodeMetaData> nodeMetaDatas; if (node.getStatus() == SolrApiNodeStatus.DELETED) { // Fake the empty node metadata for this parent deleted node NodeMetaData nodeMetaData = new NodeMetaData(); nodeMetaData.setId(node.getId()); nodeMetaData.setType(ContentModel.TYPE_DELETED); nodeMetaData.setNodeRef(new NodeRef(node.getNodeRef())); nodeMetaData.setTxnId(node.getTxnId()); nodeMetaDatas = Collections.singletonList(nodeMetaData); } else { nodeMetaDatas = coreTracker.getNodesMetaData(nmdp, MAX_RESULTS_NODES_META_DATA); } for (NodeMetaData nodeMetaData : nodeMetaDatas) { if (nodeMetaData.getTxnId() > node.getTxnId()) { // the node has moved on to a later transaction // it will be indexed later continue; } LinkedHashSet<Long> visited = new LinkedHashSet<Long>(); updateDescendantAuxDocs(nodeMetaData, overwrite, solrIndexSearcher, visited, solrIndexSearcher.getDocSet(skippingDocsQuery)); } log.debug(".. deleting"); DeleteUpdateCommand docCmd = new DeleteUpdateCommand(); docCmd.id = "LEAF-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = "AUX-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = "UNINDEXED-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = PREFIX_ERROR + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); } if ((node.getStatus() == SolrApiNodeStatus.UPDATED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) { log.info(".. updating"); NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(node.getId()); nmdp.setToNodeId(node.getId()); List<NodeMetaData> nodeMetaDatas = coreTracker.getNodesMetaData(nmdp, MAX_RESULTS_NODES_META_DATA); AddUpdateCommand leafDocCmd = new AddUpdateCommand(); leafDocCmd.overwriteCommitted = overwrite; leafDocCmd.overwritePending = overwrite; AddUpdateCommand auxDocCmd = new AddUpdateCommand(); auxDocCmd.overwriteCommitted = overwrite; auxDocCmd.overwritePending = overwrite; ArrayList<Reader> toClose = new ArrayList<Reader>(); ArrayList<File> toDelete = new ArrayList<File>(); for (NodeMetaData nodeMetaData : nodeMetaDatas) { if (nodeMetaData.getTxnId() > node.getTxnId()) { // the node has moved on to a later transaction // it will be indexed later continue; } if (mayHaveChildren(nodeMetaData)) { log.info(".. checking for path change"); BooleanQuery bQuery = new BooleanQuery(); bQuery.add(new TermQuery( new Term(QueryConstants.FIELD_DBID, NumericEncoder.encode(nodeMetaData.getId()))), Occur.MUST); bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_PARENT_ASSOC_CRC, NumericEncoder.encode(nodeMetaData.getParentAssocsCrc()))), Occur.MUST); DocSet docSet = solrIndexSearcher.getDocSet(bQuery); if (docSet.size() > 0) { log.debug("... found aux match"); } else { docSet = solrIndexSearcher.getDocSet(new TermQuery(new Term(QueryConstants.FIELD_DBID, NumericEncoder.encode(nodeMetaData.getId())))); if (docSet.size() > 0) { log.debug("... cascade updating aux doc"); LinkedHashSet<Long> visited = new LinkedHashSet<Long>(); updateDescendantAuxDocs(nodeMetaData, overwrite, solrIndexSearcher, visited, solrIndexSearcher.getDocSet(skippingDocsQuery)); } else { log.debug("... no aux doc"); } } } Map<QName, PropertyValue> properties = nodeMetaData.getProperties(); // check index control if (properties.containsKey(ContentModel.PROP_IS_INDEXED)) { StringPropertyValue pValue = (StringPropertyValue) properties .get(ContentModel.PROP_IS_INDEXED); if (pValue != null) { Boolean isIndexed = Boolean.valueOf(pValue.getValue()); if ((isIndexed != null) && (isIndexed.booleanValue() == false)) { DeleteUpdateCommand docCmd = new DeleteUpdateCommand(); docCmd.id = "LEAF-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = "AUX-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = PREFIX_ERROR + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); SolrInputDocument doc = new SolrInputDocument(); doc.addField(QueryConstants.FIELD_ID, "UNINDEXED-" + nodeMetaData.getId()); doc.addField(QueryConstants.FIELD_DBID, nodeMetaData.getId()); doc.addField(QueryConstants.FIELD_LID, nodeMetaData.getNodeRef()); doc.addField(QueryConstants.FIELD_INTXID, nodeMetaData.getTxnId()); leafDocCmd.solrDoc = doc; leafDocCmd.doc = toDocument(leafDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); if (leafDocCmd.doc != null && recordUnindexedNodes) { core.getUpdateHandler().addDoc(leafDocCmd); } long end = System.nanoTime(); coreTracker.getTrackerStats().addNodeTime(end - start); return; } } } boolean isContentIndexedForNode = true; if (properties.containsKey(ContentModel.PROP_IS_CONTENT_INDEXED)) { StringPropertyValue pValue = (StringPropertyValue) properties .get(ContentModel.PROP_IS_CONTENT_INDEXED); if (pValue != null) { Boolean isIndexed = Boolean.valueOf(pValue.getValue()); if ((isIndexed != null) && (isIndexed.booleanValue() == false)) { isContentIndexedForNode = false; } } } // Make sure any unindexed doc is removed. DeleteUpdateCommand docCmd = new DeleteUpdateCommand(); docCmd.id = "UNINDEXED-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = PREFIX_ERROR + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); SolrInputDocument doc = new SolrInputDocument(); doc.addField(QueryConstants.FIELD_ID, "LEAF-" + nodeMetaData.getId()); doc.addField(QueryConstants.FIELD_DBID, nodeMetaData.getId()); doc.addField(QueryConstants.FIELD_LID, nodeMetaData.getNodeRef()); doc.addField(QueryConstants.FIELD_INTXID, nodeMetaData.getTxnId()); for (QName propertyQname : properties.keySet()) { if (dataModel.isIndexedOrStored(propertyQname)) { PropertyValue value = properties.get(propertyQname); if (value != null) { if (value instanceof ContentPropertyValue) { addContentPropertyToDoc(doc, toClose, toDelete, nodeMetaData, propertyQname, (ContentPropertyValue) value, isContentIndexedForNode); } else if (value instanceof MLTextPropertyValue) { addMLTextPropertyToDoc(doc, propertyQname, (MLTextPropertyValue) value); } else if (value instanceof MultiPropertyValue) { MultiPropertyValue typedValue = (MultiPropertyValue) value; for (PropertyValue singleValue : typedValue.getValues()) { if (singleValue instanceof ContentPropertyValue) { addContentPropertyToDoc(doc, toClose, toDelete, nodeMetaData, propertyQname, (ContentPropertyValue) singleValue, isContentIndexedForNode); } else if (singleValue instanceof MLTextPropertyValue) { addMLTextPropertyToDoc(doc, propertyQname, (MLTextPropertyValue) singleValue); } else if (singleValue instanceof StringPropertyValue) { addStringPropertyToDoc(doc, propertyQname, (StringPropertyValue) singleValue, properties); } } } else if (value instanceof StringPropertyValue) { addStringPropertyToDoc(doc, propertyQname, (StringPropertyValue) value, properties); } } } } doc.addField(QueryConstants.FIELD_TYPE, nodeMetaData.getType().toString()); for (QName aspect : nodeMetaData.getAspects()) { doc.addField(QueryConstants.FIELD_ASPECT, aspect.toString()); } doc.addField(QueryConstants.FIELD_ISNODE, "T"); doc.addField(QueryConstants.FIELD_FTSSTATUS, "Clean"); // TODO: Place holder to test tenant queries String tenant = nodeMetaData.getTenantDomain(); if (tenant.length() > 0) { doc.addField(QueryConstants.FIELD_TENANT, nodeMetaData.getTenantDomain()); } else { doc.addField(QueryConstants.FIELD_TENANT, "_DEFAULT_"); } leafDocCmd.solrDoc = doc; leafDocCmd.doc = toDocument(leafDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); SolrInputDocument aux = createAuxDoc(nodeMetaData); auxDocCmd.solrDoc = aux; auxDocCmd.doc = toDocument(auxDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); } if (leafDocCmd.doc != null) { core.getUpdateHandler().addDoc(leafDocCmd); } if (auxDocCmd.doc != null) { core.getUpdateHandler().addDoc(auxDocCmd); } for (Reader forClose : toClose) { try { forClose.close(); } catch (IOException ioe) { } } for (File file : toDelete) { file.delete(); } } long end = System.nanoTime(); coreTracker.getTrackerStats().addNodeTime(end - start); } catch (Exception e) { // generic recovery // Add failed node marker to try later // TODO: add to reporting // TODO: Store exception for display via query // TODO: retry failed DeleteUpdateCommand docCmd = new DeleteUpdateCommand(); docCmd.id = "LEAF-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = "AUX-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); docCmd = new DeleteUpdateCommand(); docCmd.id = "UNINDEXED-" + node.getId(); docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); AddUpdateCommand leafDocCmd = new AddUpdateCommand(); leafDocCmd.overwriteCommitted = overwrite; leafDocCmd.overwritePending = overwrite; SolrInputDocument doc = new SolrInputDocument(); doc.addField(QueryConstants.FIELD_ID, PREFIX_ERROR + node.getId()); doc.addField(QueryConstants.FIELD_DBID, node.getId()); doc.addField(QueryConstants.FIELD_INTXID, node.getTxnId()); doc.addField(QueryConstants.FIELD_EXCEPTION_MESSAGE, e.getMessage()); StringWriter stringWriter = new StringWriter(4096); PrintWriter printWriter = new PrintWriter(stringWriter, true); try { e.printStackTrace(printWriter); doc.addField(QueryConstants.FIELD_EXCEPTION_STACK, stringWriter.toString()); } finally { printWriter.close(); } leafDocCmd.solrDoc = doc; leafDocCmd.doc = toDocument(leafDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); if (leafDocCmd.doc != null) { core.getUpdateHandler().addDoc(leafDocCmd); } log.warn("Node index failed and skipped for " + node.getId() + " in Tx " + node.getTxnId(), e); } finally { if (refCounted != null) { refCounted.decref(); } } }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
private void doUpdateDescendantAuxDocs(NodeMetaData parentNodeMetaData, boolean overwrite, SolrIndexSearcher solrIndexSearcher, LinkedHashSet<Long> stack, DocSet skippingDocs) throws AuthenticationException, IOException, JSONException { if ((skipDescendantAuxDocsForSpecificTypes && typesForSkippingDescendantAuxDocs.contains(parentNodeMetaData.getType())) || (skipDescendantAuxDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects()))) { return;/*from www .j av a 2 s . c om*/ } HashSet<Long> childIds = new HashSet<Long>(); if (parentNodeMetaData.getChildIds() != null) { childIds.addAll(parentNodeMetaData.getChildIds()); } BooleanQuery bQuery = new BooleanQuery(); bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_PARENT, parentNodeMetaData.getNodeRef().toString())), Occur.MUST); DocSet docSet = solrIndexSearcher.getDocSet(bQuery); ResizeableArrayList<CacheEntry> indexedByDocId = (ResizeableArrayList<CacheEntry>) solrIndexSearcher .cacheLookup(AlfrescoSolrEventListener.ALFRESCO_ARRAYLIST_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID); if (docSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) docSet; OpenBitSet openBitSet = source.getBits(); int current = -1; while ((current = openBitSet.nextSetBit(current + 1)) != -1) { if (!skippingDocs.exists(current)) { CacheEntry entry = indexedByDocId.get(current); childIds.add(entry.getDbid()); } } } else { for (DocIterator it = docSet.iterator(); it.hasNext(); /* */) { int current = it.nextDoc(); if (!skippingDocs.exists(current)) { CacheEntry entry = indexedByDocId.get(current); childIds.add(entry.getDbid()); } } } for (Long childId : childIds) { if (!shouldElementBeIgnored(childId, solrIndexSearcher, skippingDocs)) { NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setFromNodeId(childId); nmdp.setToNodeId(childId); nmdp.setIncludeAclId(true); nmdp.setIncludeAspects(true); nmdp.setIncludeChildAssociations(false); nmdp.setIncludeChildIds(true); nmdp.setIncludeNodeRef(true); nmdp.setIncludeOwner(true); nmdp.setIncludeParentAssociations(true); nmdp.setIncludePaths(true); nmdp.setIncludeProperties(false); nmdp.setIncludeType(true); nmdp.setIncludeTxnId(true); // call back to core tracker to talk to client List<NodeMetaData> nodeMetaDatas = coreTracker.getNodesMetaData(nmdp, MAX_RESULTS_NODES_META_DATA); for (NodeMetaData nodeMetaData : nodeMetaDatas) { if (mayHaveChildren(nodeMetaData)) { updateDescendantAuxDocs(nodeMetaData, overwrite, solrIndexSearcher, stack, skippingDocs); } // Avoid adding aux docs for stuff yet to be indexed or unindexed (via the index control aspect) log.info(".. checking aux doc exists in index before we update it"); Query query = new TermQuery(new Term(QueryConstants.FIELD_ID, "AUX-" + childId)); DocSet auxSet = solrIndexSearcher.getDocSet(query); if (auxSet.size() > 0) { log.debug("... cascade update aux doc " + childId); SolrInputDocument aux = createAuxDoc(nodeMetaData); AddUpdateCommand auxDocCmd = new AddUpdateCommand(); auxDocCmd.overwriteCommitted = overwrite; auxDocCmd.overwritePending = overwrite; auxDocCmd.solrDoc = aux; auxDocCmd.doc = toDocument(auxDocCmd.getSolrInputDocument(), core.getSchema(), dataModel); core.getUpdateHandler().addDoc(auxDocCmd); } else { log.debug("... no aux doc found to update " + childId); } } } } }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
private boolean shouldElementBeIgnored(long dbId, SolrIndexSearcher solrIndexSearcher, DocSet skippingDocs) throws IOException { boolean result = false; if ((skipDescendantAuxDocsForSpecificTypes && !typesForSkippingDescendantAuxDocs.isEmpty()) || (skipDescendantAuxDocsForSpecificAspects && !aspectsForSkippingDescendantAuxDocs.isEmpty())) { BooleanQuery query = new BooleanQuery(); query.add(new TermQuery(new Term(QueryConstants.FIELD_DBID, NumericEncoder.encode(dbId))), Occur.MUST); DocSet docSet = solrIndexSearcher.getDocSet(query); int index = -1; if (docSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) docSet; OpenBitSet openBitSet = source.getBits(); index = openBitSet.nextSetBit(index + 1); } else {// w w w . j a v a 2s. c o m DocIterator it = docSet.iterator(); if (it.hasNext()) { index = it.nextDoc(); } } result = (-1 != index) && skippingDocs.exists(index); } return result; }
From source file:org.alfresco.solr.LegacySolrInformationServer.java
License:Open Source License
private void deleteByQuery(SolrIndexSearcher solrIndexSearcher, Query query) throws IOException { HashSet<String> idsToDelete = new HashSet<String>(); DocSet docSet = solrIndexSearcher.getDocSet(query); if (docSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) docSet; OpenBitSet openBitSet = source.getBits(); int current = -1; while ((current = openBitSet.nextSetBit(current + 1)) != -1) { Document doc = solrIndexSearcher.doc(current, Collections.singleton(QueryConstants.FIELD_ID)); Fieldable fieldable = doc.getFieldable(QueryConstants.FIELD_ID); if (fieldable != null) { idsToDelete.add(fieldable.stringValue()); }// ww w .j a v a2s . c o m } } else { for (DocIterator it = docSet.iterator(); it.hasNext(); /* */) { Document doc = solrIndexSearcher.doc(it.nextDoc(), Collections.singleton(QueryConstants.FIELD_ID)); Fieldable fieldable = doc.getFieldable(QueryConstants.FIELD_ID); if (fieldable != null) { idsToDelete.add(fieldable.stringValue()); } } } for (String idToDelete : idsToDelete) { DeleteUpdateCommand docCmd = new DeleteUpdateCommand(); docCmd.id = idToDelete; docCmd.fromPending = true; docCmd.fromCommitted = true; core.getUpdateHandler().delete(docCmd); } }