List of usage examples for org.apache.solr.client.solrj SolrQuery getCopy
public SolrQuery getCopy()
From source file:au.org.ala.biocache.dao.SearchDAOImpl.java
License:Open Source License
/** * Writes the index fields to the supplied output stream in CSV format. * <p>/* w w w . ja v a 2s. c om*/ * DM: refactored to split the query by month to improve performance. * Further enhancements possible: * 1) Multi threaded * 2) More filtering, by year or decade.. * * @param downloadParams * @param out * @param includeSensitive * @param dd The details of the download * @param checkLimit * @param nextExecutor The ExecutorService to use to process results on different threads * @throws Exception */ @Override public ConcurrentMap<String, AtomicInteger> writeResultsFromIndexToStream( final DownloadRequestParams downloadParams, final OutputStream out, final boolean includeSensitive, final DownloadDetailsDTO dd, boolean checkLimit, final ExecutorService nextExecutor) throws Exception { expandRequestedFields(downloadParams, true); if (dd != null) { dd.resetCounts(); } long start = System.currentTimeMillis(); final ConcurrentMap<String, AtomicInteger> uidStats = new ConcurrentHashMap<>(); getServer(); try { SolrQuery solrQuery = new SolrQuery(); queryFormatUtils.formatSearchQuery(downloadParams); String dFields = downloadParams.getFields(); if (includeSensitive) { //include raw latitude and longitudes if (dFields.contains("decimalLatitude.p")) { dFields = dFields.replaceFirst("decimalLatitude.p", "sensitive_latitude,sensitive_longitude,decimalLatitude.p"); } else if (dFields.contains("decimalLatitude")) { dFields = dFields.replaceFirst("decimalLatitude", "sensitive_latitude,sensitive_longitude,decimalLatitude"); } if (dFields.contains(",locality,")) { dFields = dFields.replaceFirst(",locality,", ",locality,sensitive_locality,"); } if (dFields.contains(",locality.p,")) { dFields = dFields.replaceFirst(",locality.p,", ",locality.p,sensitive_locality,"); } } StringBuilder sb = new StringBuilder(dFields); if (!downloadParams.getExtra().isEmpty()) { sb.append(",").append(downloadParams.getExtra()); } String[] requestedFields = sb.toString().split(","); List<String>[] indexedFields; if (downloadFields == null) { //default to include everything java.util.List<String> mappedNames = new java.util.LinkedList<String>(); for (int i = 0; i < requestedFields.length; i++) mappedNames.add(requestedFields[i]); indexedFields = new List[] { mappedNames, new java.util.LinkedList<String>(), mappedNames, mappedNames, new ArrayList(), new ArrayList() }; } else { indexedFields = downloadFields.getIndexFields(requestedFields, downloadParams.getDwcHeaders(), downloadParams.getLayersServiceUrl()); } //apply custom header String[] customHeader = dd.getRequestParams().getCustomHeader().split(","); for (int i = 0; i + 1 < customHeader.length; i += 2) { for (int j = 0; j < indexedFields[0].size(); j++) { if (customHeader[i].equals(indexedFields[0].get(j))) { indexedFields[2].set(j, customHeader[i + 1]); } } for (int j = 0; j < indexedFields[4].size(); j++) { if (customHeader[i].equals(indexedFields[5].get(j))) { indexedFields[4].set(j, customHeader[i + 1]); } } } if (logger.isDebugEnabled()) { logger.debug("Fields included in download: " + indexedFields[0]); logger.debug("Fields excluded from download: " + indexedFields[1]); logger.debug("The headers in downloads: " + indexedFields[2]); logger.debug("Analysis headers: " + indexedFields[4]); logger.debug("Analysis fields: " + indexedFields[5]); } //set the fields to the ones that are available in the index String[] fields = indexedFields[0].toArray(new String[] {}); solrQuery.setFields(fields); StringBuilder qasb = new StringBuilder(); if (!"none".equals(downloadParams.getQa())) { solrQuery.addField("assertions"); if (!"all".equals(downloadParams.getQa()) && !"includeall".equals(downloadParams.getQa())) { //add all the qa fields qasb.append(downloadParams.getQa()); } } solrQuery.addField("institution_uid").addField("collection_uid").addField("data_resource_uid") .addField("data_provider_uid"); solrQuery.setQuery(downloadParams.getFormattedQuery()); solrQuery.setFacetMinCount(1); solrQuery.setFacetLimit(-1); //get the assertion facets to add them to the download fields boolean getAssertionsFromFacets = "all".equals(downloadParams.getQa()) || "includeall".equals(downloadParams.getQa()); SolrQuery monthAssertionsQuery = getAssertionsFromFacets ? solrQuery.getCopy().addFacetField("month", "assertions") : solrQuery.getCopy().addFacetField("month"); if (getAssertionsFromFacets) { //set the order for the facet to be based on the index - this will force the assertions to be returned in the same order each time //based on alphabetical sort. The number of QA's may change between searches so we can't guarantee that the order won't change monthAssertionsQuery.add("f.assertions.facet.sort", "index"); } QueryResponse facetQuery = runSolrQuery(monthAssertionsQuery, downloadParams.getFormattedFq(), 0, 0, "score", "asc"); //set the totalrecords for the download details dd.setTotalRecords(facetQuery.getResults().getNumFound()); //use a separately configured and smaller limit when output will be unzipped final long maxDownloadSize; if (MAX_DOWNLOAD_SIZE > unzippedLimit && out instanceof OptionalZipOutputStream && ((OptionalZipOutputStream) out).getType() == OptionalZipOutputStream.Type.unzipped) { maxDownloadSize = unzippedLimit; } else { maxDownloadSize = MAX_DOWNLOAD_SIZE; } if (checkLimit && dd.getTotalRecords() < maxDownloadSize) { checkLimit = false; } //get the month facets to add them to the download fields get the assertion facets. List<Count> splitByFacet = null; for (FacetField facet : facetQuery.getFacetFields()) { if (facet.getName().equals("assertions") && facet.getValueCount() > 0) { qasb.append(getQAFromFacet(facet)); } if (facet.getName().equals("month") && facet.getValueCount() > 0) { splitByFacet = facet.getValues(); } } if ("includeall".equals(downloadParams.getQa())) { qasb = getAllQAFields(); } String qas = qasb.toString(); //include sensitive fields in the header when the output will be partially sensitive final String[] sensitiveFields; final String[] notSensitiveFields; if (dd.getSensitiveFq() != null) { List<String>[] sensitiveHdr = downloadFields.getIndexFields(sensitiveSOLRHdr, downloadParams.getDwcHeaders(), downloadParams.getLayersServiceUrl()); //header for the output file indexedFields[2].addAll(sensitiveHdr[2]); //lookup for fields from sensitive queries sensitiveFields = org.apache.commons.lang3.ArrayUtils.addAll( indexedFields[0].toArray(new String[] {}), sensitiveHdr[0].toArray(new String[] {})); //use general fields when sensitive data is not permitted notSensitiveFields = org.apache.commons.lang3.ArrayUtils .addAll(indexedFields[0].toArray(new String[] {}), notSensitiveSOLRHdr); } else { sensitiveFields = new String[0]; notSensitiveFields = fields; } //add analysis headers indexedFields[2].addAll(indexedFields[4]); final String[] analysisFields = indexedFields[5].toArray(new String[0]); final String[] qaFields = qas.equals("") ? new String[] {} : qas.split(","); String[] qaTitles = downloadFields.getHeader(qaFields, false, false); String[] header = org.apache.commons.lang3.ArrayUtils.addAll(indexedFields[2].toArray(new String[] {}), qaTitles); //retain output header fields and field names for inclusion of header info in the download StringBuilder infoFields = new StringBuilder("infoFields"); for (String h : indexedFields[3]) infoFields.append(",").append(h); for (String h : qaFields) infoFields.append(",").append(h); StringBuilder infoHeader = new StringBuilder("infoHeaders"); for (String h : header) infoHeader.append(",").append(h); String info = infoFields.toString(); while (info.contains(",,")) info = info.replace(",,", ","); uidStats.put(info, new AtomicInteger(-1)); String hdr = infoHeader.toString(); while (hdr.contains(",,")) hdr = hdr.replace(",,", ","); uidStats.put(hdr, new AtomicInteger(-2)); //construct correct RecordWriter based on the supplied fileType final RecordWriterError rw = downloadParams.getFileType().equals("csv") ? new CSVRecordWriter(out, header, downloadParams.getSep(), downloadParams.getEsc()) : (downloadParams.getFileType().equals("tsv") ? new TSVRecordWriter(out, header) : new ShapeFileRecordWriter(tmpShapefileDir, downloadParams.getFile(), out, (String[]) ArrayUtils.addAll(fields, qaFields))); // Requirement to be able to propagate interruptions to all other threads for this execution // Doing this via this variable final AtomicBoolean interruptFound = dd != null ? dd.getInterrupt() : new AtomicBoolean(false); // Create a fixed length blocking queue for buffering results before they are written // This also creates a push-back effect to throttle the results generating threads // when it fills and offers to it are delayed until the writer consumes elements from the queue final BlockingQueue<String[]> queue = new ArrayBlockingQueue<>(resultsQueueLength); // Create a sentinel that we can check for reference equality to signal the end of the queue final String[] sentinel = new String[0]; // An implementation of RecordWriter that adds to an in-memory queue final RecordWriter concurrentWrapper = new RecordWriter() { private AtomicBoolean finalised = new AtomicBoolean(false); private AtomicBoolean finalisedComplete = new AtomicBoolean(false); @Override public void write(String[] nextLine) { try { if (Thread.currentThread().isInterrupted() || interruptFound.get() || finalised.get()) { finalise(); return; } while (!queue.offer(nextLine, writerTimeoutWaitMillis, TimeUnit.MILLISECONDS)) { if (Thread.currentThread().isInterrupted() || interruptFound.get() || finalised.get()) { finalise(); break; } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); interruptFound.set(true); if (logger.isDebugEnabled()) { logger.debug( "Queue failed to accept the next record due to a thread interrupt, calling finalise the cleanup: ", e); } // If we were interrupted then we should call finalise to cleanup finalise(); } } @Override public void finalise() { if (finalised.compareAndSet(false, true)) { try { // Offer the sentinel at least once, even when the thread is interrupted while (!queue.offer(sentinel, writerTimeoutWaitMillis, TimeUnit.MILLISECONDS)) { // If the thread is interrupted then the queue may not have any active consumers, // so don't loop forever waiting for capacity in this case // The hard shutdown phase will use queue.clear to ensure that the // sentinel gets onto the queue at least once if (Thread.currentThread().isInterrupted() || interruptFound.get()) { break; } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); interruptFound.set(true); if (logger.isDebugEnabled()) { logger.debug( "Queue failed to accept the sentinel in finalise due to a thread interrupt: ", e); } } finally { finalisedComplete.set(true); } } } @Override public boolean finalised() { return finalisedComplete.get(); } }; // A single thread that consumes elements put onto the queue until it sees the sentinel, finalising after the sentinel or an interrupt Runnable writerRunnable = new Runnable() { @Override public void run() { try { long counter = 0; while (true) { counter = counter + 1; if (Thread.currentThread().isInterrupted() || interruptFound.get()) { break; } String[] take = queue.take(); // Sentinel object equality check to see if we are done if (take == sentinel || Thread.currentThread().isInterrupted() || interruptFound.get()) { break; } // Otherwise write to the wrapped record writer rw.write(take); //test for errors. This can contain a flush so only test occasionally if (counter % resultsQueueLength == 0 && rw.hasError()) { throw RecordWriterException.newRecordWriterException(dd, downloadParams, true, rw); } } } catch (RecordWriterException e) { //no trace information is available to print for these errors logger.error(e.getMessage()); interruptFound.set(true); } catch (InterruptedException e) { Thread.currentThread().interrupt(); interruptFound.set(true); } catch (Exception e) { // Reuse interruptFound variable to signal that the writer had issues interruptFound.set(true); logger.error("Download writer failed.", e); } finally { rw.finalise(); } } }; Thread writerThread = new Thread(writerRunnable); writerThread.start(); try { if (rw instanceof ShapeFileRecordWriter) { dd.setHeaderMap(((ShapeFileRecordWriter) rw).getHeaderMappings()); } //order the query by _docid_ for faster paging solrQuery.addSortField("_docid_", ORDER.asc); //for each month create a separate query that pages through 500 records per page List<SolrQuery> queries = new ArrayList<SolrQuery>(); if (splitByFacet != null) { for (Count facet : splitByFacet) { if (facet.getCount() > 0) { SolrQuery splitByFacetQuery; //do not add remainderQuery here if (facet.getName() != null) { splitByFacetQuery = solrQuery.getCopy() .addFilterQuery(facet.getFacetField().getName() + ":" + facet.getName()); splitByFacetQuery.setFacet(false); queries.add(splitByFacetQuery); } } } if (splitByFacet.size() > 0) { SolrQuery remainderQuery = solrQuery.getCopy() .addFilterQuery("-" + splitByFacet.get(0).getFacetField().getName() + ":[* TO *]"); queries.add(0, remainderQuery); } } else { queries.add(0, solrQuery); } //split into sensitive and non-sensitive queries when // - not including all sensitive values // - there is a sensitive fq final List<SolrQuery> sensitiveQ = new ArrayList<SolrQuery>(); if (!includeSensitive && dd.getSensitiveFq() != null) { sensitiveQ.addAll( splitQueries(queries, dd.getSensitiveFq(), sensitiveSOLRHdr, notSensitiveSOLRHdr)); } //Set<Future<Integer>> futures = new HashSet<Future<Integer>>(); final AtomicInteger resultsCount = new AtomicInteger(0); final boolean threadCheckLimit = checkLimit; List<Callable<Integer>> solrCallables = new ArrayList<>(queries.size()); // execute each query, writing the results to stream for (final SolrQuery splitByFacetQuery : queries) { // define a thread Callable<Integer> solrCallable = new Callable<Integer>() { @Override public Integer call() throws Exception { int startIndex = 0; // Randomise the wakeup time so they don't all wakeup on a periodic cycle long localThrottle = throttle + Math.round(Math.random() * throttle); String[] fq = downloadParams.getFormattedFq(); if (splitByFacetQuery.getFilterQueries() != null && splitByFacetQuery.getFilterQueries().length > 0) { if (fq == null) { fq = new String[0]; } fq = org.apache.commons.lang3.ArrayUtils.addAll(fq, splitByFacetQuery.getFilterQueries()); } QueryResponse qr = runSolrQuery(splitByFacetQuery, fq, downloadBatchSize, startIndex, "_docid_", "asc"); AtomicInteger recordsForThread = new AtomicInteger(0); if (logger.isDebugEnabled()) { logger.debug( splitByFacetQuery.getQuery() + " - results: " + qr.getResults().size()); } while (qr != null && !qr.getResults().isEmpty() && !interruptFound.get()) { if (logger.isDebugEnabled()) { logger.debug( "Start index: " + startIndex + ", " + splitByFacetQuery.getQuery()); } int count = 0; if (sensitiveQ.contains(splitByFacetQuery)) { count = processQueryResults(uidStats, sensitiveFields, qaFields, concurrentWrapper, qr, dd, threadCheckLimit, resultsCount, maxDownloadSize, analysisFields); } else { // write non-sensitive values into sensitive fields when not authorised for their sensitive values count = processQueryResults(uidStats, notSensitiveFields, qaFields, concurrentWrapper, qr, dd, threadCheckLimit, resultsCount, maxDownloadSize, analysisFields); } recordsForThread.addAndGet(count); startIndex += downloadBatchSize; // we have already set the Filter query the first time the query was constructed // rerun with the same params but different startIndex if (!threadCheckLimit || resultsCount.get() < maxDownloadSize) { if (!threadCheckLimit) { // throttle the download by sleeping Thread.sleep(localThrottle); } qr = runSolrQuery(splitByFacetQuery, null, downloadBatchSize, startIndex, "_docid_", "asc"); } else { qr = null; } } return recordsForThread.get(); } }; solrCallables.add(solrCallable); } List<Future<Integer>> futures = new ArrayList<>(solrCallables.size()); for (Callable<Integer> nextCallable : solrCallables) { futures.add(nextExecutor.submit(nextCallable)); } // Busy wait because we need to be able to respond to an interrupt on any callable // and propagate it to all of the others for this particular query // Because the executor service is shared to prevent too many concurrent threads being run, // this requires a busy wait loop on the main thread to monitor state boolean waitAgain = false; do { waitAgain = false; for (Future<Integer> future : futures) { if (!future.isDone()) { // Wait again even if an interrupt flag is set, as it may have been set partway through the iteration // The calls to future.cancel will occur next time if the interrupt is setup partway through an iteration waitAgain = true; // If one thread finds an interrupt it is propagated to others using the interruptFound AtomicBoolean if (interruptFound.get()) { future.cancel(true); } } } // Don't trigger the timeout interrupt if we don't have to wait again as we are already done at this point if (waitAgain && (System.currentTimeMillis() - start) > downloadMaxTime) { interruptFound.set(true); break; } if (waitAgain) { Thread.sleep(downloadCheckBusyWaitSleep); } } while (waitAgain); AtomicInteger totalDownload = new AtomicInteger(0); for (Future<Integer> future : futures) { if (future.isDone()) { totalDownload.addAndGet(future.get()); } else { // All incomplete futures that survived the loop above are cancelled here future.cancel(true); } } long finish = System.currentTimeMillis(); long timeTakenInSecs = (finish - start) / 1000; if (timeTakenInSecs <= 0) timeTakenInSecs = 1; if (logger.isInfoEnabled()) { logger.info("Download of " + resultsCount + " records in " + timeTakenInSecs + " seconds. Record/sec: " + resultsCount.intValue() / timeTakenInSecs); } } finally { try { // Once we get here, we need to finalise starting at the concurrent wrapper, // as there are no more non-sentinel records to be added to the queue // This eventually triggers finalisation of the underlying writer when the queue empties // This is a soft shutdown, and hence we wait below for this stage to complete in normal circumstances // Note, this blocks for writerTimeoutWaitMillis trying to legitimately add the sentinel to the end of the queue // We force the sentinel to be added in the hard shutdown phase below concurrentWrapper.finalise(); } finally { try { // Track the current time right now so we can abort after downloadMaxCompletionTime milliseconds in this phase final long completionStartTime = System.currentTimeMillis(); // Busy wait check for finalised to be called in the RecordWriter or something is interrupted // By this stage, there are at maximum download.internal.queue.size items remaining (default 1000) while (writerThread.isAlive() && !writerThread.isInterrupted() && !interruptFound.get() && !Thread.currentThread().isInterrupted() && !rw.finalised() && !((System.currentTimeMillis() - completionStartTime) > downloadMaxCompletionTime)) { Thread.sleep(downloadCheckBusyWaitSleep); } } finally { try { // Attempt all actions that could trigger the writer thread to finalise, as by this stage we are in hard shutdown mode // Signal that we are in hard shutdown mode interruptFound.set(true); // Add the sentinel or clear the queue and try again until it gets onto the queue // We are in hard shutdown mode, so only priority is that the queue either // gets the sentinel or the thread is interrupted to clean up resources while (!queue.offer(sentinel)) { queue.clear(); } // Interrupt the single writer thread writerThread.interrupt(); // Explicitly call finalise on the RecordWriter as a backup // In normal circumstances it is called via the sentinel or the interrupt // This will not block if finalise has been called previously in the current three implementations rw.finalise(); } finally { if (rw != null && rw.hasError()) { throw RecordWriterException.newRecordWriterException(dd, downloadParams, true, rw); } else { // Flush whatever output was still pending for more deterministic debugging out.flush(); } } } } } } catch (SolrServerException ex) { logger.error("Problem communicating with SOLR server while processing download. " + ex.getMessage(), ex); } return uidStats; }
From source file:au.org.ala.biocache.dao.SearchDAOImpl.java
License:Open Source License
/** * Split a list of queries by a fq./* w w w .java 2s . co m*/ */ private List<SolrQuery> splitQueries(List<SolrQuery> queries, String fq, String[] fqFields, String[] notFqFields) { List<SolrQuery> notFQ = new ArrayList<SolrQuery>(); List<SolrQuery> fQ = new ArrayList<SolrQuery>(); for (SolrQuery query : queries) { SolrQuery nsq = query.getCopy().addFilterQuery("-(" + fq + ")"); if (notFqFields != null) { for (String field : notFqFields) nsq.addField(field); } notFQ.add(nsq); SolrQuery sq = query.getCopy().addFilterQuery(fq); if (fqFields != null) { for (String field : fqFields) sq.addField(field); } fQ.add(sq); } queries.clear(); queries.addAll(notFQ); queries.addAll(fQ); return fQ; }
From source file:de.qaware.chronix.solr.client.stream.SolrStreamingService.java
License:Apache License
private void initStreamingService(SolrQuery query, SolrClient connection) { SolrQuery solrQuery = query.getCopy(); solrQuery.setRows(0);/*w w w. ja v a2s . co m*/ //we do not need any data from the server expect the total number of found documents solrQuery.setFields(""); try { QueryResponse response = connection.query(solrQuery); nrOfAvailableTimeSeries = response.getResults().getNumFound(); queryStart = (long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_START_LONG); queryEnd = (long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_END_LONG); } catch (SolrServerException | IOException e) { LOGGER.error("SolrServerException occurred while querying server.", e); } }
From source file:de.qaware.chronix.storage.solr.stream.SolrStreamingService.java
License:Apache License
private void initialStream(SolrQuery query, SolrClient connection) { try {/*from w w w .j a v a 2s . co m*/ SolrQuery solrQuery = query.getCopy(); solrQuery.setRows(nrOfTimeSeriesPerBatch); solrQuery.setStart(currentDocumentCount); solrStreamingHandler.init(nrOfTimeSeriesPerBatch, currentDocumentCount); QueryResponse response = connection.queryAndStreamResponse(solrQuery, solrStreamingHandler); nrOfAvailableTimeSeries = response.getResults().getNumFound(); queryStart = 0;//(long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_START_LONG); queryEnd = Long.MAX_VALUE;//(long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_END_LONG); needStream = false; } catch (SolrServerException | IOException e) { LOGGER.error("SolrServerException occurred while querying server.", e); } }
From source file:edu.ku.brc.sgr.MatchableIndexedId.java
License:Open Source License
@Override public MatchResults doMatch(SolrServer server, SolrQuery baseQuery) { final SolrQuery query = baseQuery.getCopy(); query.setQuery("id:" + id); final QueryResponse resp; try {/*from w w w . j a v a 2 s .c o m*/ resp = server.query(query); } catch (SolrServerException e) { // TODO Auto-generated catch block throw new RuntimeException(e); } final SolrDocumentList docs = resp.getResults(); final float maxScore = (docs != null) ? docs.getMaxScore() : 0.0f; final ImmutableList.Builder<Match> msBuilder = ImmutableList.builder(); if (docs != null) { for (SolrDocument doc : docs) { float score = (Float) doc.getFieldValue("score"); SGRRecord match = SGRRecord.fromSolrDocument(doc); msBuilder.add(new Match(match, score)); } } return new MatchResults(id, resp.getQTime(), maxScore, msBuilder.build()); }
From source file:edu.ku.brc.sgr.MatchableRecord.java
License:Open Source License
@Override public MatchResults doMatch(SolrServer server, SolrQuery baseQuery) { final ModifiableSolrParams query = baseQuery.getCopy().set(MoreLikeThisParams.DOC_SUPPLIED, true); final MoreLikeThisDocumentRequest req = new MoreLikeThisDocumentRequest(query) .addDocument(record.asSolrDocument()); final QueryResponse resp; try {/* w w w . j a v a2 s. c o m*/ resp = req.process(server); } catch (Exception e) { // TODO Auto-generated catch block throw new RuntimeException(e); } final SolrDocumentList docs = resp.getResults(); Map<String, String> explain = resp.getExplainMap(); if (explain == null) explain = Maps.newHashMap(); final ImmutableList.Builder<Match> msBuilder = ImmutableList.builder(); final float maxScore = (docs != null) ? docs.getMaxScore() : 0.0f; if (docs != null) { for (SolrDocument doc : docs) { float score = (Float) doc.getFieldValue("score"); SGRRecord match = SGRRecord.fromSolrDocument(doc); String explained = explain.get(match.id); msBuilder.add(new Match(match, score, explained)); } } return new MatchResults(record.id, resp.getQTime(), maxScore, msBuilder.build()); }
From source file:edu.unc.lib.dl.ui.service.SolrQueryLayerService.java
License:Apache License
public void getChildrenCounts(List<BriefObjectMetadata> resultList, AccessGroupSet accessGroups, String countName, String queryAddendum, SolrQuery baseQuery) { long startTime = System.currentTimeMillis(); if (resultList == null || resultList.size() == 0) return;// w ww .java2 s . c o m String ancestorPathField = solrSettings.getFieldName(SearchFieldKeys.ANCESTOR_PATH.name()); SolrQuery solrQuery; if (baseQuery == null) { // Create a base query since we didn't receive one solrQuery = new SolrQuery(); StringBuilder query = new StringBuilder("*:*"); try { // Add access restrictions to query addAccessRestrictions(query, accessGroups); } catch (AccessRestrictionException e) { // If the user doesn't have any access groups, they don't have access to anything, return null. LOG.error(e.getMessage()); return; } solrQuery.setStart(0); solrQuery.setRows(0); solrQuery.setQuery(query.toString()); } else { // Starting from a base query solrQuery = baseQuery.getCopy(); // Make sure we aren't returning any normal results solrQuery.setRows(0); // Remove all facet fields so we are only getting ancestor path if (solrQuery.getFacetFields() != null) { for (String facetField : solrQuery.getFacetFields()) { solrQuery.removeFacetField(facetField); } } } if (queryAddendum != null) { solrQuery.setQuery(solrQuery.getQuery() + " AND " + queryAddendum); } solrQuery.setFacet(true); solrQuery.setFacetMinCount(1); solrQuery.addFacetField(ancestorPathField); Integer countPageSize; try { countPageSize = new Integer(searchSettings.getProperty("search.facet.countPageSize")); } catch (NumberFormatException e) { countPageSize = 20; } solrQuery.add("f." + ancestorPathField + ".facet.limit", Integer.toString(Integer.MAX_VALUE)); // Sort by value rather than count so that earlier tiers will come first in case the result gets cut off solrQuery.setFacetSort("index"); java.util.Map<Integer, StringBuilder> tierQueryMap = new java.util.HashMap<Integer, StringBuilder>(); java.util.Map<Integer, List<BriefObjectMetadata>> containerMap = new java.util.HashMap<Integer, List<BriefObjectMetadata>>(); // Pare the list of ids we are searching for and assigning counts to down to just containers for (BriefObjectMetadata metadataObject : resultList) { if (metadataObject.getPath() != null && metadataObject.getContentModel() != null && metadataObject.getContentModel().contains(ContentModelHelper.Model.CONTAINER.toString())) { CutoffFacetNode highestTier = metadataObject.getPath().getHighestTierNode(); StringBuilder tierQuery = tierQueryMap.get(highestTier.getTier()); List<BriefObjectMetadata> containerObjects = containerMap.get(highestTier.getTier()); if (tierQuery == null) { tierQuery = new StringBuilder(); tierQueryMap.put(highestTier.getTier(), tierQuery); containerObjects = new ArrayList<BriefObjectMetadata>(); containerMap.put(highestTier.getTier(), containerObjects); } if (tierQuery.length() == 0) { tierQuery.append(ancestorPathField).append(":("); } else { tierQuery.append(" OR "); } tierQuery.append(SolrSettings.sanitize(highestTier.getSearchValue())).append(",*"); containerObjects.add(metadataObject); // If there are a lot of results, then do a partial lookup if (containerObjects.size() >= countPageSize) { tierQuery.append(")"); this.executeChildrenCounts(tierQuery, containerObjects, solrQuery, countName, highestTier.getTier()); LOG.info("Partial query done at " + System.currentTimeMillis() + " (" + (System.currentTimeMillis() - startTime) + ")"); containerMap.remove(highestTier.getTier()); tierQueryMap.remove(highestTier.getTier()); } } } Iterator<java.util.Map.Entry<Integer, StringBuilder>> queryIt = tierQueryMap.entrySet().iterator(); while (queryIt.hasNext()) { java.util.Map.Entry<Integer, StringBuilder> tierQueryEntry = queryIt.next(); tierQueryEntry.getValue().append(')'); this.executeChildrenCounts(tierQueryEntry.getValue(), containerMap.get(tierQueryEntry.getKey()), solrQuery, countName, tierQueryEntry.getKey()); } LOG.info("Child count query done at " + System.currentTimeMillis() + " (" + (System.currentTimeMillis() - startTime) + ")"); }
From source file:edu.unc.lib.dl.ui.service.SolrQueryLayerService.java
License:Apache License
/** * Retrieves results for populating a hierarchical browse view. Supports all the regular navigation available to * searches. Results contain child counts for each item (all items returned are containers), and a map containing the * number of nested subcontainers per container. Children counts are retrieved based on facet counts. * /*from ww w. j ava2s. c o m*/ * @param browseRequest * @return */ public HierarchicalBrowseResultResponse getHierarchicalBrowseResults(HierarchicalBrowseRequest browseRequest) { AccessGroupSet accessGroups = browseRequest.getAccessGroups(); SearchState browseState = (SearchState) browseRequest.getSearchState().clone(); HierarchicalBrowseResultResponse browseResults = new HierarchicalBrowseResultResponse(); CutoffFacet rootPath = null; BriefObjectMetadataBean rootNode = null; if (browseRequest.getRootPid() != null) { rootNode = getObjectById( new SimpleIdRequest(browseRequest.getRootPid(), browseRequest.getAccessGroups())); if (rootNode != null) { rootPath = rootNode.getPath(); browseState.getFacets().put(SearchFieldKeys.ANCESTOR_PATH.name(), rootPath); browseResults.setSelectedContainer(rootNode); } } // Default the ancestor path to the collections object so we always have a root if (rootNode == null) { rootPath = (CutoffFacet) browseState.getFacets().get(SearchFieldKeys.ANCESTOR_PATH.name()); if (rootPath == null) { rootPath = new CutoffFacet(SearchFieldKeys.ANCESTOR_PATH.name(), "1," + this.collectionsPid.getPid()); browseState.getFacets().put(SearchFieldKeys.ANCESTOR_PATH.name(), rootPath); } rootNode = getObjectById(new SimpleIdRequest(rootPath.getSearchKey(), browseRequest.getAccessGroups())); } boolean rootIsAStub = rootNode == null; if (rootIsAStub) { // Parent is not found, but children are, so make a stub for the parent. rootNode = new BriefObjectMetadataBean(); rootNode.setId(rootPath.getSearchKey()); rootNode.setAncestorPathFacet(rootPath); } SearchState hierarchyState = searchStateFactory.createHierarchyListSearchState(); // Use the ancestor path facet from the state where we will have set a default value hierarchyState.getFacets().put(SearchFieldKeys.ANCESTOR_PATH.name(), rootPath); hierarchyState.setRowsPerPage(0); SearchRequest hierarchyRequest = new SearchRequest(hierarchyState, accessGroups, false); SolrQuery baseQuery = this.generateSearch(hierarchyRequest); // Get the set of all applicable containers SolrQuery hierarchyQuery = baseQuery.getCopy(); hierarchyQuery.setRows(new Integer(searchSettings.getProperty("search.results.maxBrowsePerPage"))); // Reusable query segment for limiting the results to just the depth asked for StringBuilder cutoffQuery = new StringBuilder(); cutoffQuery.append('!').append(solrSettings.getFieldName(SearchFieldKeys.ANCESTOR_PATH.name())).append(":"); cutoffQuery.append(rootPath.getHighestTier() + browseRequest.getRetrievalDepth()); cutoffQuery.append(searchSettings.facetSubfieldDelimiter).append('*'); hierarchyQuery.addFilterQuery(cutoffQuery.toString()); SearchResultResponse results; try { results = this.executeSearch(hierarchyQuery, hierarchyState, false, false); browseResults.setSearchResultResponse(results); } catch (SolrServerException e) { LOG.error("Error while getting container results for hierarchical browse results", e); return null; } // Add the root node into the result set browseResults.getResultList().add(0, rootNode); if (browseRequest.isRetrieveFacets() && browseRequest.getSearchState().getFacetsToRetrieve() != null) { SearchState facetState = (SearchState) browseState.clone(); facetState.setRowsPerPage(0); SearchRequest facetRequest = new SearchRequest(facetState, browseRequest.getAccessGroups(), true); SolrQuery facetQuery = this.generateSearch(facetRequest); try { SearchResultResponse facetResponse = this.executeSearch(facetQuery, facetState, true, false); browseResults.setFacetFields(facetResponse.getFacetFields()); } catch (SolrServerException e) { LOG.warn("Failed to retrieve facet results for " + facetQuery.toString(), e); } } // Don't need to manipulate the container list any further unless either the root is a real record or there are // subcontainers if (!rootIsAStub || results.getResultCount() > 0) { // Get the children counts per container SearchRequest filteredChildrenRequest = new SearchRequest(browseState, browseRequest.getAccessGroups(), true); this.getChildrenCounts(results.getResultList(), accessGroups, "child", null, this.generateSearch(filteredChildrenRequest)); this.getChildrenCounts(results.getResultList(), accessGroups, "containers", "contentModel:" + SolrSettings.sanitize(ContentModelHelper.Model.CONTAINER.toString()), this.generateSearch(filteredChildrenRequest)); try { // If anything that constituted a search is in the request then trim out possible empty folders if (browseState.getFacets().size() > 1 || browseState.getRangeFields().size() > 0 || browseState.getSearchFields().size() > 0 || browseState.getAccessTypeFilter() != null) { // Get the list of any direct matches for the current query browseResults .setMatchingContainerPids(this.getDirectContainerMatches(browseState, accessGroups)); // Remove all containers that are not direct matches for the user's query and have 0 children browseResults.removeContainersWithoutContents(); } } catch (SolrServerException e) { LOG.error("Error while getting children counts for hierarchical browse", e); return null; } } // Retrieve normal item search results, which are restricted to a max number per page if (browseRequest.isIncludeFiles() && browseState.getRowsPerPage() > 0) { browseState.getResourceTypes().add(searchSettings.resourceTypeFile); SearchState fileSearchState = new SearchState(browseState); List<String> resourceTypes = new ArrayList<String>(); resourceTypes.add(searchSettings.resourceTypeFile); fileSearchState.setResourceTypes(resourceTypes); CutoffFacet ancestorPath = (CutoffFacet) fileSearchState.getFacets() .get(SearchFieldKeys.ANCESTOR_PATH.name()); ancestorPath.setCutoff(rootPath.getHighestTier() + 1); fileSearchState.setFacetsToRetrieve(null); SearchRequest fileSearchRequest = new SearchRequest(fileSearchState, browseRequest.getAccessGroups()); SearchResultResponse fileResults = this.getSearchResults(fileSearchRequest); browseResults.populateItemResults(fileResults.getResultList()); } browseResults.generateResultTree(); return browseResults; }
From source file:eu.europeana.core.BeanQueryModelFactory.java
License:EUPL
SolrQuery addHiddenQueryFilters(SolrQuery solrQuery, Map<String, String[]> params) { SolrQuery dCopy = solrQuery.getCopy(); if (params != null && params.containsKey("hqf")) { final String[] hqf_fields = SolrQueryUtil.getFilterQueriesAsPhrases(params.get("hqf")); for (String hqf_field : hqf_fields) { dCopy.addFilterQuery(hqf_field); }/*from w w w. ja va 2 s.com*/ } final PortalTheme theme = ThemeFilter.getTheme(); if (theme != null && !theme.getHiddenQueryFilters().isEmpty()) { final String[] themeHqf = SolrQueryUtil .getFilterQueriesAsPhrases(theme.getHiddenQueryFilters().split(",")); for (String qf : themeHqf) { dCopy.addFilterQuery(qf); } } dCopy.setFilterQueries(SolrQueryUtil.getFilterQueriesAsOrQueries(dCopy, ThemeFilter.getTheme().getRecordDefinition().getFacetMap())); return dCopy; }
From source file:org.apache.nutch.indexwriter.solr.TestSolrJ.java
License:Apache License
/** * query the example/*w w w . j a v a 2 s .c om*/ */ @Test public void testQuery() throws Exception { SolrClient client = new HttpSolrClient.Builder(serverUrl).build(); // Empty the database... client.deleteByQuery("*:*");// delete everything! // Now add something... SolrInputDocument doc = new SolrInputDocument(); String docID = "1112211111"; doc.addField("id", docID, 1.0f); doc.addField("name", "my name!", 1.0f); assertEquals(null, doc.getField("foo")); assertTrue(doc.getField("name").getValue() != null); UpdateResponse upres = client.add(doc); // System.out.println( "ADD:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); upres = client.commit(true, true); // System.out.println( "COMMIT:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); upres = client.optimize(true, true); // System.out.println( "OPTIMIZE:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); SolrQuery query = new SolrQuery(); query.setQuery("id:" + docID); QueryResponse response = client.query(query); assertEquals(docID, response.getResults().get(0).getFieldValue("id")); // Now add a few docs for facet testing... List<SolrInputDocument> docs = new ArrayList<>(); SolrInputDocument doc2 = new SolrInputDocument(); doc2.addField("id", "2", 1.0f); doc2.addField("inStock", true, 1.0f); doc2.addField("price", 2, 1.0f); doc2.addField("timestamp_dt", new java.util.Date(), 1.0f); docs.add(doc2); SolrInputDocument doc3 = new SolrInputDocument(); doc3.addField("id", "3", 1.0f); doc3.addField("inStock", false, 1.0f); doc3.addField("price", 3, 1.0f); doc3.addField("timestamp_dt", new java.util.Date(), 1.0f); docs.add(doc3); SolrInputDocument doc4 = new SolrInputDocument(); doc4.addField("id", "4", 1.0f); doc4.addField("inStock", true, 1.0f); doc4.addField("price", 4, 1.0f); doc4.addField("timestamp_dt", new java.util.Date(), 1.0f); docs.add(doc4); SolrInputDocument doc5 = new SolrInputDocument(); doc5.addField("id", "5", 1.0f); doc5.addField("inStock", false, 1.0f); doc5.addField("price", 5, 1.0f); doc5.addField("timestamp_dt", new java.util.Date(), 1.0f); docs.add(doc5); upres = client.add(docs); // System.out.println( "ADD:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); upres = client.commit(true, true); // System.out.println( "COMMIT:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); upres = client.optimize(true, true); // System.out.println( "OPTIMIZE:"+upres.getResponse() ); assertEquals(0, upres.getStatus()); query = new SolrQuery("*:*"); query.addFacetQuery("price:[* TO 2]"); query.addFacetQuery("price:[2 TO 4]"); query.addFacetQuery("price:[5 TO *]"); query.addFacetField("inStock"); query.addFacetField("price"); query.addFacetField("timestamp_dt"); query.removeFilterQuery("inStock:true"); response = client.query(query); assertEquals(0, response.getStatus()); assertEquals(5, response.getResults().getNumFound()); assertEquals(3, response.getFacetQuery().size()); assertEquals(2, response.getFacetField("inStock").getValueCount()); assertEquals(4, response.getFacetField("price").getValueCount()); // test a second query, test making a copy of the main query SolrQuery query2 = query.getCopy(); query2.addFilterQuery("inStock:true"); response = client.query(query2); assertEquals(1, query2.getFilterQueries().length); assertEquals(0, response.getStatus()); assertEquals(2, response.getResults().getNumFound()); assertFalse(query.getFilterQueries() == query2.getFilterQueries()); // sanity check round tripping of params... query = new SolrQuery("foo"); query.addFilterQuery("{!field f=inStock}true"); query.addFilterQuery("{!term f=name}hoss"); query.addFacetQuery("price:[* TO 2]"); query.addFacetQuery("price:[2 TO 4]"); response = client.query(query); assertTrue("echoed params are not a NamedList: " + response.getResponseHeader().get("params").getClass(), response.getResponseHeader().get("params") instanceof NamedList); NamedList echo = (NamedList) response.getResponseHeader().get("params"); List values = null; assertEquals("foo", echo.get("q")); assertTrue("echoed fq is not a List: " + echo.get("fq").getClass(), echo.get("fq") instanceof List); values = (List) echo.get("fq"); assertEquals(2, values.size()); assertEquals("{!field f=inStock}true", values.get(0)); assertEquals("{!term f=name}hoss", values.get(1)); assertTrue("echoed facet.query is not a List: " + echo.get("facet.query").getClass(), echo.get("facet.query") instanceof List); values = (List) echo.get("facet.query"); assertEquals(2, values.size()); assertEquals("price:[* TO 2]", values.get(0)); assertEquals("price:[2 TO 4]", values.get(1)); }