List of usage examples for org.apache.solr.client.solrj SolrQuery getQuery
public String getQuery()
From source file:net.yacy.peers.Protocol.java
License:Open Source License
protected static int solrQuery(final SearchEvent event, final SolrQuery solrQuery, final int offset, final int count, final Seed target, final int partitions, final Blacklist blacklist) { if (event.query.getQueryGoal().getQueryString(false) == null || event.query.getQueryGoal().getQueryString(false).length() == 0) { return -1; // we cannot query solr only with word hashes, there is no clear text string }/* w w w . j a v a 2 s . c o m*/ event.addExpectedRemoteReferences(count); if (partitions > 0) solrQuery.set("partitions", partitions); solrQuery.setStart(offset); solrQuery.setRows(count); // set highlighting query attributes if (event.query.contentdom == Classification.ContentDomain.TEXT || event.query.contentdom == Classification.ContentDomain.ALL) { solrQuery.setHighlight(true); solrQuery.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH); //solrQuery.setHighlightRequireFieldMatch(); solrQuery.setHighlightSimplePost("</b>"); solrQuery.setHighlightSimplePre("<b>"); solrQuery.setHighlightSnippets(5); for (CollectionSchema field : snippetFields) solrQuery.addHighlightField(field.getSolrFieldName()); } else { solrQuery.setHighlight(false); } boolean localsearch = target == null || target.equals(event.peers.mySeed()); Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>( event.query.facetfields.size()); Map<String, LinkedHashSet<String>> snippets = new HashMap<String, LinkedHashSet<String>>(); // this will be a list of urlhash-snippet entries final QueryResponse[] rsp = new QueryResponse[] { null }; final SolrDocumentList[] docList = new SolrDocumentList[] { null }; String ip = target.getIP(); {// encapsulate expensive solr QueryResponse object if (localsearch && !Switchboard.getSwitchboard() .getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { // search the local index try { SolrConnector sc = event.getQuery().getSegment().fulltext().getDefaultConnector(); if (!sc.isClosed()) { rsp[0] = sc.getResponseByParams(solrQuery); docList[0] = rsp[0].getResults(); } } catch (final Throwable e) { Network.log.info("SEARCH failed (solr), localpeer (" + e.getMessage() + ")", e); return -1; } } else { try { final boolean myseed = target == event.peers.mySeed(); if (!myseed && !target.getFlagSolrAvailable()) { // skip if peer.dna has flag that last try resulted in error Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); return -1; } final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(ip); final int solrtimeout = Switchboard.getSwitchboard() .getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); Thread remoteRequest = new Thread() { @Override public void run() { this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + target.hash + ")"); try { RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr", solrtimeout); // this is a 'patch configuration' which considers 'solr' as default collection try { SolrConnector solrConnector = new RemoteSolrConnector(instance, myseed ? true : target.getVersion() >= 1.63, "solr"); if (!solrConnector.isClosed()) try { rsp[0] = solrConnector.getResponseByParams(solrQuery); docList[0] = rsp[0].getResults(); } catch (Throwable e) { } finally { solrConnector.close(); } } catch (Throwable ee) { } finally { instance.close(); } } catch (Throwable eee) { } } }; remoteRequest.start(); remoteRequest.join(solrtimeout); // just wait until timeout appears if (remoteRequest.isAlive()) { try { remoteRequest.interrupt(); } catch (Throwable e) { } Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " does not answer (time-out)"); target.setFlagSolrAvailable(false || myseed); return -1; // give up, leave remoteRequest abandoned. } // no need to close this here because that sends a commit to remote solr which is not wanted here } catch (final Throwable e) { Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " (" + e.getMessage() + ")"); target.setFlagSolrAvailable(false || localsearch); return -1; } } if (rsp[0] == null || docList[0] == null) { Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null"); target.setFlagSolrAvailable(false || localsearch); return -1; } // evaluate facets for (String field : event.query.facetfields) { FacetField facet = rsp[0].getFacetField(field); ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator); List<Count> values = facet == null ? null : facet.getValues(); if (values == null) continue; for (Count ff : values) { int c = (int) ff.getCount(); if (c == 0) continue; if (ff.getName().length() == 0) continue; // facet entry without text is not useful result.set(ff.getName(), c); } if (result.size() > 0) facets.put(field, result); } // evaluate snippets Map<String, Map<String, List<String>>> rawsnippets = rsp[0].getHighlighting(); // a map from the urlhash to a map with key=field and value = list of snippets if (rawsnippets != null) { nextsnippet: for (Map.Entry<String, Map<String, List<String>>> re : rawsnippets.entrySet()) { Map<String, List<String>> rs = re.getValue(); for (CollectionSchema field : snippetFields) { if (rs.containsKey(field.getSolrFieldName())) { List<String> s = rs.get(field.getSolrFieldName()); if (s.size() > 0) { LinkedHashSet<String> ls = new LinkedHashSet<String>(); ls.addAll(s); snippets.put(re.getKey(), ls); continue nextsnippet; } } } // no snippet found :( --we don't assign a value here by default; that can be done as an evaluation outside this method } } rsp[0] = null; } // evaluate result if (docList == null || docList[0].size() == 0) { Network.log.info("SEARCH (solr), returned 0 out of 0 documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()); return 0; } List<URIMetadataNode> container = new ArrayList<URIMetadataNode>(); Network.log.info("SEARCH (solr), returned " + docList[0].size() + " out of " + docList[0].getNumFound() + " documents and " + facets.size() + " facets " + facets.keySet().toString() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); int term = count; Collection<SolrInputDocument> docs; if (event.addResultsToLocalIndex) { // only needed to store remote results docs = new ArrayList<SolrInputDocument>(docList[0].size()); } else docs = null; for (final SolrDocument doc : docList[0]) { if (term-- <= 0) { break; // do not process more that requested (in case that evil peers fill us up with rubbish) } // get one single search result if (doc == null) { continue; } URIMetadataNode urlEntry = new URIMetadataNode(doc); if (blacklist.isListed(BlacklistType.SEARCH, urlEntry.url())) { if (Network.log.isInfo()) { if (localsearch) { Network.log.info("local search (solr): filtered blacklisted url " + urlEntry.url()); } else { Network.log.info("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); } } continue; // block with blacklist } final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker .urlInAcceptedDomain(urlEntry.url()); if (urlRejectReason != null) { if (Network.log.isInfo()) { if (localsearch) { Network.log.info("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")"); } else { Network.log.info("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); } } continue; // reject url outside of our domain } // passed all checks, store url if (!localsearch) { // put the remote documents to the local index. We must convert the solr document to a solr input document: if (event.addResultsToLocalIndex) { final SolrInputDocument sid = event.query.getSegment().fulltext().getDefaultConfiguration() .toSolrInputDocument(doc); // the input document stays untouched because it contains top-level cloned objects docs.add(sid); // will be stored to index, and is a full solr document, can be added to firstseen event.query.getSegment().setFirstSeenTime(urlEntry.hash(), Math.min(urlEntry.moddate().getTime(), System.currentTimeMillis())); } // after this conversion we can remove the largest and not used field text_t and synonyms_sxt from the document // because that goes into a search cache and would take a lot of memory in the search cache //doc.removeFields(CollectionSchema.text_t.getSolrFieldName()); doc.removeFields(CollectionSchema.synonyms_sxt.getSolrFieldName()); ResultURLs.stack(ASCII.String(urlEntry.url().hash()), urlEntry.url().getHost(), event.peers.mySeed().hash.getBytes(), UTF8.getBytes(target.hash), EventOrigin.QUERIES); } // add the url entry to the word indexes container.add(urlEntry); } final int dls = docList[0].size(); final int numFound = (int) docList[0].getNumFound(); docList[0].clear(); docList[0] = null; if (localsearch) { event.addNodes(container, facets, snippets, true, "localpeer", numFound); event.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.info( "local search (solr): localpeer sent " + container.size() + "/" + numFound + " references"); } else { if (event.addResultsToLocalIndex) { for (SolrInputDocument doc : docs) { event.query.getSegment().putDocument(doc); } docs.clear(); docs = null; } event.addNodes(container, facets, snippets, false, target.getName() + "/" + target.hash, numFound); event.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.info("remote search (solr): peer " + target.getName() + " sent " + (container.size() == 0 ? 0 : container.size()) + "/" + numFound + " references"); } return dls; }
From source file:net.yacy.peers.RemoteSearch.java
License:Open Source License
public static Thread solrRemoteSearch(final SearchEvent event, final SolrQuery solrQuery, final int start, final int count, final Seed targetPeer, final int partitions, final Blacklist blacklist) { assert solrQuery != null; // check own peer status if (event.peers.mySeed() == null) { return null; }//ww w. ja v a2s .co m // prepare threads Thread solr = new Thread() { @Override public void run() { this.setName("RemoteSearch.solrRemoteSearch(" + solrQuery.getQuery() + " to " + (targetPeer == null ? "myself" : targetPeer.hash) + ")"); int urls = 0; try { event.oneFeederStarted(); urls = Protocol.solrQuery(event, solrQuery, start, count, targetPeer == null ? event.peers.mySeed() : targetPeer, partitions, blacklist); if (urls >= 0) { // urls is an array of url hashes. this is only used for log output event.peers.mySeed().incRI(urls); event.peers.mySeed().incRU(urls); } else { if (targetPeer != null) { Network.log.info("REMOTE SEARCH - no answer from remote peer " + targetPeer.hash + ":" + targetPeer.getName()); } } } catch (final Exception e) { ConcurrentLog.logException(e); } finally { event.oneFeederTerminated(); } } }; /*if (targetPeer == null) solr.run(); else*/ solr.start(); return solr; }
From source file:net.yacy.search.AutoSearch.java
License:Open Source License
/** * Calls one peer for search results of the current query and adds it to the * local index. Depending on peers SolrAvailable flag the a solr query or * opensearch/rss query is used.//from w w w .ja v a 2 s. c om * * @param seed the peer to ask */ private void processSingleTarget(Seed seed) { ConcurrentLog.fine(AutoSearch.class.getName(), "ask " + seed.getIP() + " " + seed.getName() + " for query=" + currentQuery); if (seed.getFlagSolrAvailable()) { // do a solr query SolrDocumentList docList = null; SolrQuery solrQuery = new SolrQuery(); // use remote defaults and ranking (to query their index right) solrQuery.set(CommonParams.Q, currentQuery + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)"); // except this yacy special solrQuery.set("q.op", "AND"); // except ... no one word matches please solrQuery.set(CommonParams.ROWS, "20"); this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")"); try { RemoteInstance instance = new RemoteInstance( "http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection try { SolrConnector solrConnector = new RemoteSolrConnector(instance, true, null); if (!solrConnector.isClosed()) { try { QueryResponse rsp = solrConnector.getResponseByParams(solrQuery); docList = rsp.getResults(); } catch (Throwable e) { } finally { solrConnector.close(); } } } catch (Throwable ee) { } finally { instance.close(); } if (docList != null) { for (SolrDocument d : docList) { sb.index.fulltext() .putDocument(sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(d)); this.gotresults++; } ConcurrentLog.info(AutoSearch.class.getName(), "added " + docList.size() + " results from " + seed.getName() + " to index for solrquery=" + currentQuery); } } catch (Throwable eee) { } } else { // do a yacysearch.rss query final String rssSearchServiceURL = "http://" + seed.getPublicAddress(seed.getIP()) + "/yacysearch.rss"; try { RSSFeed feed = loadSRURSS(rssSearchServiceURL, currentQuery, 0, 20, CacheStrategy.IFFRESH, false, // just local, as we ask others too ClientIdentification.yacyInternetCrawlerAgent); final List<DigestURL> urls = new ArrayList<DigestURL>(); for (final MultiProtocolURL entry : feed.getLinks()) { urls.add(new DigestURL(entry, (byte[]) null)); this.gotresults++; } sb.addToIndex(urls, null, "AutoSearch", null, true); ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery); } catch (IOException ex) { ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName()); } } }
From source file:nl.knaw.dans.common.solr.SolrServerWrapper.java
License:Apache License
public void deleteByQuery(SearchRequest request) throws SearchEngineException { try {/*from ww w.j a v a 2s.co m*/ SolrQuery query = SolrQueryRequestConverter.convert(request); LOGGER.info("Solr: deleting by Query " + query.toString()); server.deleteByQuery(query.getQuery()); } catch (SolrServerException e) { throw new SearchEngineException(e); } catch (IOException e) { throw new SearchEngineException(e); } }
From source file:nl.knaw.huygens.facetedsearch.AbstractSolrServer.java
License:Open Source License
private Map<String, Object> getSearchData(ElaborateSearchParameters sp, String[] facetFields, SolrQuery query, String[] fieldsToReturn) throws IndexException { Map<String, Object> data = Maps.newHashMap(); data.put("term", query.getQuery()); try {/*w w w . j a v a 2 s. com*/ Log.info("query=\n{}", query); QueryResponse response = server.query(query); Log.debug("response='{}'", response); SolrDocumentList documents = response.getResults(); data.put(KEY_NUMFOUND, documents.getNumFound()); Map<String, Map<String, List<String>>> highlighting = response.getHighlighting(); List<String> ids = Lists.newArrayList(); List<Map<String, Object>> results = Lists.newArrayList(); int occurrences = 0; for (SolrDocument document : documents) { String docId = document.getFieldValue(SolrFields.DOC_ID).toString(); ids.add(docId); Map<String, List<String>> map = (Map<String, List<String>>) ((highlighting == null) ? ImmutableMap.of() : highlighting.get(docId)); Map<String, Object> result = entryView(document, fieldsToReturn, map, sp.getTextFieldsToSearch()); results.add(result); for (Integer integer : ((Map<String, Integer>) result.get("terms")).values()) { occurrences += integer; } } data.put("ids", ids); data.put("results", results); data.put("occurrences", occurrences); data.put("facets", getFacetCountList(sp, facetFields, response)); } catch (SolrServerException e) { Log.error(e.getMessage()); throw new IndexException(e.getMessage()); } data.put("solrquery", query.toString()); return data; }
From source file:nl.knaw.huygens.timbuctoo.index.solr.SolrQueryMatcher.java
License:Open Source License
public SolrQueryMatcher withQuery(String query) { this.addMatcher(new PropertyEqualityMatcher<SolrQuery, String>("query", query) { @Override/*from w w w. java 2 s . c o m*/ protected String getItemValue(SolrQuery item) { return item.getQuery(); } }); return this; }
From source file:org.ala.dao.FulltextSearchDaoImplSolr.java
License:Open Source License
private QueryResponse getSolrQueryResponse(SolrQuery solrQuery, String[] filterQuery, Integer pageSize, Integer startIndex, String sortField, String sortDirection) throws Exception { if (logger.isDebugEnabled()) { logger.debug(solrQuery.getQuery()); }// ww w . ja v a2 s . c om // set the facet query if set addFqs(solrQuery, filterQuery); solrQuery.setRows(pageSize); solrQuery.setStart(startIndex); solrQuery.setSortField(sortField, ORDER.valueOf(sortDirection)); // do the Solr search return solrUtils.getSolrServer().query(solrQuery); // can throw exception }
From source file:org.ala.dao.FulltextSearchDaoImplSolr.java
License:Open Source License
/** * Re-usable method for performing SOLR searches - takes SolrQuery input * * @param solrQuery/*from w w w . j av a 2s . co m*/ * @param filterQuery * @param pageSize * @param startIndex * @param sortField * @param sortDirection * @return * @throws SolrServerException */ private SearchResultsDTO doSolrQuery(SolrQuery solrQuery, String[] filterQuery, Integer pageSize, Integer startIndex, String sortField, String sortDirection) throws Exception { SearchResultsDTO searchResults = new SearchResultsDTO(); QueryResponse qr = getSolrQueryResponse(solrQuery, filterQuery, pageSize, startIndex, sortField, sortDirection); //process results SolrDocumentList sdl = qr.getResults(); List<FacetField> facets = qr.getFacetFields(); List<SearchDTO> results = new ArrayList<SearchDTO>(); List<FacetResultDTO> facetResults = new ArrayList<FacetResultDTO>(); searchResults.setTotalRecords(sdl.getNumFound()); searchResults.setStartIndex(sdl.getStart()); searchResults.setPageSize(pageSize); searchResults.setStatus("OK"); searchResults.setSort(sortField); searchResults.setDir(sortDirection); searchResults.setQuery(solrQuery.getQuery()); // populate SOLR search results if (!sdl.isEmpty()) { for (SolrDocument doc : sdl) { if (IndexedTypes.TAXON.toString().equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createTaxonConceptFromIndex(qr, doc)); } else if (IndexedTypes.COLLECTION.toString() .equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createCollectionFromIndex(qr, doc)); } else if (IndexedTypes.INSTITUTION.toString() .equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createInstitutionFromIndex(qr, doc)); } else if (IndexedTypes.DATAPROVIDER.toString() .equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createDataProviderFromIndex(qr, doc)); } else if (IndexedTypes.DATASET.toString() .equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createDatasetFromIndex(qr, doc)); } else if (IndexedTypes.REGION.toString().equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createRegionFromIndex(qr, doc)); } else if (IndexedTypes.WORDPRESS.toString() .equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createWordpressFromIndex(qr, doc)); } else if (IndexedTypes.LAYERS.toString().equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { results.add(createLayerFromIndex(qr, doc)); } else { results.add(createSearchDTOFromIndex(qr, doc)); } } } else { logger.debug("No results for query."); } searchResults.setResults(results); // populate SOLR facet results if (facets != null) { for (FacetField facet : facets) { List<FacetField.Count> facetEntries = facet.getValues(); if ((facetEntries != null) && (facetEntries.size() > 0)) { ArrayList<FieldResultDTO> r = new ArrayList<FieldResultDTO>(); for (FacetField.Count fcount : facetEntries) { r.add(new FieldResultDTO(fcount.getName(), fcount.getCount())); } FacetResultDTO fr = new FacetResultDTO(facet.getName(), r); facetResults.add(fr); } } } searchResults.setFacetResults(facetResults); // The query result is stored in its original format so that all the information // returned is available later on if needed searchResults.setQr(qr); return searchResults; }
From source file:org.ala.dao.FulltextSearchDaoImplSolr.java
License:Open Source License
/** * @see org.ala.dao.FulltextSearchDao#getAutoCompleteList *///from w ww . j ava2 s. co m public List<AutoCompleteDTO> getAutoCompleteList(String value, String[] filterQuery, IndexedTypes indexType, boolean gsOnly, int maxTerms) { if (StringUtils.trimToNull(value) != null && value.length() >= 3) { List<AutoCompleteDTO> items = new ArrayList<AutoCompleteDTO>(); QueryResponse qr; SolrQuery solrQuery = null; try { SolrDocumentList sdl = null; //attempt to find the autocomplete solrQuery = buildAutoConcatMatchQuery(value, filterQuery, indexType, gsOnly, maxTerms); qr = solrUtils.getSolrServer().query(solrQuery); sdl = qr.getResults(); if (sdl != null && !sdl.isEmpty()) { for (SolrDocument doc : sdl) { if (IndexedTypes.TAXON.toString().equalsIgnoreCase((String) doc.getFieldValue("idxtype"))) { AutoCompleteDTO dto = createAutoCompleteFromIndex(qr, doc, value); if (dto != null && !dto.getMatchedNames().isEmpty()) { items.add(createAutoCompleteFromIndex(qr, doc, value)); } } } } } catch (Exception e) { e.printStackTrace(); logger.error(e); logger.error("***** " + e.toString() + " , queryString: " + solrQuery.getQuery()); } return items; } return new ArrayList<AutoCompleteDTO>(); }
From source file:org.ambraproject.search.service.SolrSearchService.java
License:Apache License
/** * Execute a Solr search composed from the contents of the * <code>SearchParameters.unformattedQuery</code> property. * The query is filtered by the journal and category fields also contained in the * <code>searchParameters</code> parameter. No filter is created for date ranges, since that * is assumed to be contained in <code>SearchParameters.unformattedQuery</code>. * * @param searchParameters Contains all the parameters necessary to execute a search against * the Solr query engine/*from ww w. j av a 2 s .c o m*/ * @return A subset (determined by <code>SearchParameters.startPage</code> and * <code>SearchParameters.pageSize</code> of the results of the Solr query generated from the * contents of the <code>searchParameters</code> parameter * @throws ApplicationException Thrown during failed interactions with the Solr Server */ public SearchResultSinglePage advancedSearch(SearchParameters searchParameters) throws ApplicationException { SearchParameters sp = cleanStrings(searchParameters); // Does not impact unformattedQuery field. if (log.isDebugEnabled()) { log.debug("Solr Search performed on the unformattedSearch String: " + searchParameters.getUnformattedQuery().trim()); } SolrQuery query = createQuery(null, sp.getStartPage(), sp.getPageSize(), false); query.setQuery(searchParameters.getUnformattedQuery().trim()); SolrQuery journalFacetQuery = createJournalFacetQuery(query.getQuery(), false); setFilters(query, sp, true); setFilters(journalFacetQuery, sp, false); setSort(query, sp); FacetField journals = facetSearch(journalFacetQuery, "cross_published_journal_key"); SearchResultSinglePage results = search(query.setQuery(searchParameters.getUnformattedQuery().trim())); results.setJournalFacet(facetCountsToHashMap(journals)); return results; }