List of usage examples for org.apache.solr.common.params CommonParams Q
String Q
To view the source code for org.apache.solr.common.params CommonParams Q.
Click Source Link
From source file:net.yacy.cora.federate.solr.connector.RemoteSolrConnector.java
License:Open Source License
@Override public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException { // during the solr query we set the thread name to the query string to get more debugging info in thread dumps String q = params.get(CommonParams.Q); String fq = params.get(CommonParams.FQ); String threadname = Thread.currentThread().getName(); if (q != null) Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", fq = " + fq)); QueryRequest request = new QueryRequest(params); ResponseParser responseParser = useBinaryResponseWriter ? new BinaryResponseParser() : new XMLResponseParser(); request.setResponseParser(responseParser); long t = System.currentTimeMillis(); NamedList<Object> result = null; try {/*w w w .ja v a 2s.c o m*/ result = this.server.request(request); } catch (final Throwable e) { //ConcurrentLog.logException(e); throw new IOException(e.getMessage()); /* Log.logException(e); server = instance.getServer(this.corename); super.init(server); try { result = server.request(request); } catch (final Throwable e1) { throw new IOException(e1.getMessage()); } */ } QueryResponse response = new QueryResponse(result, this.server); response.setElapsedTime(System.currentTimeMillis() - t); if (q != null) Thread.currentThread().setName(threadname); return response; }
From source file:net.yacy.cora.federate.solr.connector.SolrServerConnector.java
License:Open Source License
/** * get the solr document list from a query response * This differs from getResponseByParams in such a way that it does only create the fields of the response but * never search snippets and there are also no facets generated. * @param params/*from w ww. j ava 2 s .c om*/ * @return * @throws IOException * @throws SolrException */ @Override public SolrDocumentList getDocumentListByParams(ModifiableSolrParams params) throws IOException { if (this.server == null) throw new IOException("server disconnected"); // during the solr query we set the thread name to the query string to get more debugging info in thread dumps String q = params.get(CommonParams.Q); String fq = params.get(CommonParams.FQ); String sort = params.get(CommonParams.SORT); String fl = params.get(CommonParams.FL); String threadname = Thread.currentThread().getName(); QueryResponse rsp; int retry = 0; Throwable error = null; while (retry++ < 10) { try { if (q != null) Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", fq = " + fq) + (sort == null ? "" : ", sort = " + sort) + "; retry = " + retry + "; fl = " + fl); // for debugging in Threaddump rsp = this.server.query(params); if (q != null) Thread.currentThread().setName(threadname); if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for q=" + q); return rsp.getResults(); } catch (final SolrServerException e) { error = e; } catch (final Throwable e) { error = e; clearCaches(); // prevent further OOM if this was caused by OOM } ConcurrentLog.severe("SolrServerConnector", "Failed to query remote Solr: " + error.getMessage() + ", query:" + q + (fq == null ? "" : ", fq = " + fq)); try { Thread.sleep(1000); } catch (InterruptedException e) { } } throw new IOException("Error executing query", error); }
From source file:net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter.java
License:Open Source License
@Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { NamedList<?> values = rsp.getValues(); assert values.get("responseHeader") != null; assert values.get("response") != null; writer.write(//from www . ja va 2 s . c o m "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\n<head>\n"); writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"all\" href=\"/env/base.css\" />\n"); writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"screen\" href=\"/env/style.css\" />\n"); SolrParams params = request.getOriginalParams(); String grep = params.get("grep"); String query = ""; String q = params.get(CommonParams.Q); if (q == null) q = ""; int p = q.indexOf(':'); if (p >= 0) { int r = q.charAt(p + 1) == '"' ? q.indexOf(p + 2, '"') : q.indexOf(' '); if (r < 0) r = q.length(); query = q.substring(p + 1, r); if (query.length() > 0) { if (query.charAt(0) == '"') query = query.substring(1); if (query.charAt(query.length() - 1) == '"') query = query.substring(0, query.length() - 1); } } if (grep == null && query.length() > 0) grep = query; if (grep.length() > 0) { if (grep.charAt(0) == '"') grep = grep.substring(1); if (grep.charAt(grep.length() - 1) == '"') grep = grep.substring(0, grep.length() - 1); } NamedList<Object> paramsList = params.toNamedList(); paramsList.remove("wt"); String xmlquery = dqp.matcher("/solr/select?" + SolrParams.toSolrParams(paramsList).toString()) .replaceAll("%22"); DocList response = ((ResultContext) values.get("response")).docs; final int sz = response.size(); if (sz > 0) { SolrIndexSearcher searcher = request.getSearcher(); DocIterator iterator = response.iterator(); IndexSchema schema = request.getSchema(); String h1 = "Document Grep for query \"" + query + "\" and grep phrase \"" + grep + "\""; writer.write("<title>" + h1 + "</title>\n</head><body>\n<h1>" + h1 + "</h1>\n"); writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n"); writer.write( "<span>This search result can also be retrieved as XML. Click the API icon to see an example call to the search rss API.</span></div>\n"); for (int i = 0; i < sz; i++) { int id = iterator.nextDoc(); Document doc = searcher.doc(id, DEFAULT_FIELD_LIST); LinkedHashMap<String, String> tdoc = HTMLResponseWriter.translateDoc(schema, doc); String sku = tdoc.get(CollectionSchema.sku.getSolrFieldName()); String title = tdoc.get(CollectionSchema.title.getSolrFieldName()); String text = tdoc.get(CollectionSchema.text_t.getSolrFieldName()); ArrayList<String> sentences = new ArrayList<String>(); if (title != null) sentences.add(title); SentenceReader sr = new SentenceReader(text); StringBuilder line; while (sr.hasNext()) { line = sr.next(); if (line.length() > 0) sentences.add(line.toString()); } writeDoc(writer, sku, sentences, grep); } } else { writer.write("<title>No Document Found</title>\n</head><body>\n"); } writer.write("</body></html>\n"); }
From source file:net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter.java
License:Open Source License
@Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { assert rsp.getValues().get("responseHeader") != null; assert rsp.getValues().get("response") != null; long start = System.currentTimeMillis(); SimpleOrderedMap<Object> responseHeader = (SimpleOrderedMap<Object>) rsp.getResponseHeader(); DocList response = ((ResultContext) rsp.getValues().get("response")).docs; @SuppressWarnings("unchecked") SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) rsp.getValues().get("highlighting"); Map<String, LinkedHashSet<String>> snippets = OpensearchResponseWriter.highlighting(highlighting); Map<Object, Object> context = request.getContext(); // parse response header ResHead resHead = new ResHead(); NamedList<?> val0 = (NamedList<?>) responseHeader.get("params"); resHead.rows = Integer.parseInt((String) val0.get(CommonParams.ROWS)); resHead.offset = response.offset(); // equal to 'start' resHead.numFound = response.matches(); //resHead.df = (String) val0.get("df"); //resHead.q = (String) val0.get("q"); //resHead.wt = (String) val0.get("wt"); //resHead.status = (Integer) responseHeader.get("status"); //resHead.QTime = (Integer) responseHeader.get("QTime"); //resHead.maxScore = response.maxScore(); // write header writer.write(XML_START);//from w w w . j a va 2s . com String query = request.getParams().get("originalQuery"); String site = getContextString(context, "site", ""); String sort = getContextString(context, "sort", ""); String client = getContextString(context, "client", ""); String ip = getContextString(context, "ip", ""); String access = getContextString(context, "access", ""); String entqr = getContextString(context, "entqr", ""); OpensearchResponseWriter.solitaireTag(writer, "TM", Long.toString(System.currentTimeMillis() - start)); OpensearchResponseWriter.solitaireTag(writer, "Q", query); paramTag(writer, "sort", sort); paramTag(writer, "output", "xml_no_dtd"); paramTag(writer, "ie", "UTF-8"); paramTag(writer, "oe", "UTF-8"); paramTag(writer, "client", client); paramTag(writer, "q", query); paramTag(writer, "site", site); paramTag(writer, "start", Integer.toString(resHead.offset)); paramTag(writer, "num", Integer.toString(resHead.rows)); paramTag(writer, "ip", ip); paramTag(writer, "access", access); // p - search only public content, s - search only secure content, a - search all content, both public and secure paramTag(writer, "entqr", entqr); // query expansion policy; (entqr=1) -- Uses only the search appliance's synonym file, (entqr=1) -- Uses only the search appliance's synonym file, (entqr=3) -- Uses both standard and local synonym files. // body introduction final int responseCount = response.size(); writer.write("<RES SN=\"" + (resHead.offset + 1) + "\" EN=\"" + (resHead.offset + responseCount) + "\">"); writer.write(lb); // The index (1-based) of the first and last search result returned in this result set. writer.write("<M>" + resHead.numFound + "</M>"); writer.write(lb); // The estimated total number of results for the search. writer.write("<FI/>"); writer.write(lb); // Indicates that document filtering was performed during this search. int nextStart = resHead.offset + responseCount; int nextNum = Math.min(resHead.numFound - nextStart, responseCount < resHead.rows ? 0 : resHead.rows); int prevStart = resHead.offset - resHead.rows; if (prevStart >= 0 || nextNum > 0) { writer.write("<NB>"); if (prevStart >= 0) { writer.write("<PU>"); XML.escapeCharData("/gsa/search?q=" + request.getParams().get(CommonParams.Q) + "&site=" + site + "&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access + "&sort=" + sort + "&start=" + prevStart + "&sa=N", writer); // a relative URL pointing to the NEXT results page. writer.write("</PU>"); } if (nextNum > 0) { writer.write("<NU>"); XML.escapeCharData("/gsa/search?q=" + request.getParams().get(CommonParams.Q) + "&site=" + site + "&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access + "&sort=" + sort + "&start=" + nextStart + "&num=" + nextNum + "&sa=N", writer); // a relative URL pointing to the NEXT results page. writer.write("</NU>"); } writer.write("</NB>"); } writer.write(lb); // parse body SolrIndexSearcher searcher = request.getSearcher(); DocIterator iterator = response.iterator(); String urlhash = null; for (int i = 0; i < responseCount; i++) { int id = iterator.nextDoc(); Document doc = searcher.doc(id, SOLR_FIELDS); List<IndexableField> fields = doc.getFields(); // pre-scan the fields to get the mime-type String mime = ""; for (IndexableField value : fields) { String fieldName = value.name(); if (CollectionSchema.content_type.getSolrFieldName().equals(fieldName)) { mime = value.stringValue(); break; } } // write the R header for a search result writer.write("<R N=\"" + (resHead.offset + i + 1) + "\"" + (i == 1 ? " L=\"2\"" : "") + (mime != null && mime.length() > 0 ? " MIME=\"" + mime + "\"" : "") + ">"); writer.write(lb); //List<String> texts = new ArrayList<String>(); List<String> descriptions = new ArrayList<String>(); List<String> collections = new ArrayList<String>(); int size = 0; boolean title_written = false; // the solr index may contain several; we take only the first which should be the visible tag in <title></title> String title = null; for (IndexableField value : fields) { String fieldName = value.name(); // apply generic matching rule String stag = field2tag.get(fieldName); if (stag != null) { OpensearchResponseWriter.solitaireTag(writer, stag, value.stringValue()); continue; } // if the rule is not generic, use the specific here if (CollectionSchema.id.getSolrFieldName().equals(fieldName)) { urlhash = value.stringValue(); continue; } if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { OpensearchResponseWriter.solitaireTag(writer, GSAToken.U.name(), value.stringValue()); OpensearchResponseWriter.solitaireTag(writer, GSAToken.UE.name(), value.stringValue()); continue; } if (CollectionSchema.title.getSolrFieldName().equals(fieldName) && !title_written) { title = value.stringValue(); OpensearchResponseWriter.solitaireTag(writer, GSAToken.T.name(), highlight(title, query)); //texts.add(value.stringValue()); title_written = true; continue; } if (CollectionSchema.description_txt.getSolrFieldName().equals(fieldName)) { descriptions.add(value.stringValue()); //texts.adds(description); continue; } if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName)) { Date d = new Date(Long.parseLong(value.stringValue())); writer.write("<FS NAME=\"date\" VALUE=\"" + HeaderFramework.formatGSAFS(d) + "\"/>\n"); //OpensearchResponseWriter.solitaireTag(writer, GSAToken.CACHE_LAST_MODIFIED.getSolrFieldName(), HeaderFramework.formatRFC1123(d)); //texts.add(value.stringValue()); continue; } if (CollectionSchema.load_date_dt.getSolrFieldName().equals(fieldName)) { Date d = new Date(Long.parseLong(value.stringValue())); OpensearchResponseWriter.solitaireTag(writer, GSAToken.CRAWLDATE.name(), HeaderFramework.formatRFC1123(d)); //texts.add(value.stringValue()); continue; } if (CollectionSchema.size_i.getSolrFieldName().equals(fieldName)) { size = value.stringValue() != null && value.stringValue().length() > 0 ? Integer.parseInt(value.stringValue()) : -1; continue; } if (CollectionSchema.collection_sxt.getSolrFieldName().equals(fieldName)) { collections.add(value.stringValue()); continue; } //System.out.println("superfluous field: " + fieldName + ": " + value.stringValue()); // this can be avoided setting the enableLazyFieldLoading = false in solrconfig.xml } // compute snippet from texts LinkedHashSet<String> snippet = urlhash == null ? null : snippets.get(urlhash); OpensearchResponseWriter.removeSubsumedTitle(snippet, title); OpensearchResponseWriter.solitaireTag(writer, GSAToken.S.name(), snippet == null || snippet.size() == 0 ? (descriptions.size() > 0 ? descriptions.get(0) : "") : OpensearchResponseWriter.getLargestSnippet(snippet)); OpensearchResponseWriter.solitaireTag(writer, GSAToken.GD.name(), descriptions.size() > 0 ? descriptions.get(0) : ""); String cols = collections.toString(); if (collections.size() > 0) OpensearchResponseWriter.solitaireTag(writer, "COLS" /*SPECIAL!*/, collections.size() > 1 ? cols.substring(1, cols.length() - 1).replaceAll(" ", "") : collections.get(0)); writer.write("<HAS><L/><C SZ=\""); writer.write(Integer.toString(size / 1024)); writer.write("k\" CID=\""); writer.write(urlhash); writer.write("\" ENC=\"UTF-8\"/></HAS>\n"); if (YaCyVer == null) YaCyVer = yacyVersion.thisVersion().getName() + "/" + Switchboard.getSwitchboard().peers.mySeed().hash; OpensearchResponseWriter.solitaireTag(writer, GSAToken.ENT_SOURCE.name(), YaCyVer); OpensearchResponseWriter.closeTag(writer, "R"); } writer.write("</RES>"); writer.write(lb); writer.write(XML_STOP); }
From source file:net.yacy.http.servlets.GSAsearchServlet.java
License:Open Source License
/** * from here copy of htroot/gsa/gsasearchresult.java * with modification to use HttpServletRequest instead of (yacy) RequestHeader *//*w w w.j a v a 2 s .co m*/ public static void respond(final HttpServletRequest header, final Switchboard sb, final OutputStream out) { // remember the peer contact for peer statistics String clientip = header.getHeader(HeaderFramework.CONNECTION_PROP_CLIENTIP); if (clientip == null) clientip = "<unknown>"; // read an artificial header addendum String userAgent = header.getHeader(HeaderFramework.USER_AGENT); if (userAgent == null) userAgent = "<unknown>"; sb.peers.peerActions.setUserAgent(clientip, userAgent); // --- handled by Servlet securityHandler // check if user is allowed to search (can be switched in /ConfigPortal.html) boolean authenticated = header.isUserInRole(UserDB.AccessRight.ADMIN_RIGHT.toString()); //sb.adminAuthenticated(header) >= 2; // final boolean searchAllowed = authenticated || sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true); // if (!searchAllowed) return null; // create post serverObjects post = new serverObjects(); post.put(CommonParams.Q, ""); post.put("num", "0"); // convert servletrequest parameter to old style serverObjects map Map<String, String[]> map = header.getParameterMap(); Iterator<Map.Entry<String, String[]>> it = map.entrySet().iterator(); while (it.hasNext()) { Map.Entry<String, String[]> param = it.next(); post.put(param.getKey(), param.getValue()); // hint: post.put uses String[] for String value anyways } ConcurrentLog.info("GSA Query", post.toString()); sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time // rename post fields according to result style //post.put(CommonParams.Q, post.remove("q")); // same as solr //post.put(CommonParams.START, post.remove("start")); // same as solr //post.put(, post.remove("client"));//required, example: myfrontend //post.put(, post.remove("output"));//required, example: xml,xml_no_dtd String originalQuery = post.get(CommonParams.Q, ""); post.put("originalQuery", originalQuery); // get a solr query string QueryGoal qg = new QueryGoal(originalQuery); List<String> solrFQ = qg.collectionTextFilterQuery(false); StringBuilder solrQ = qg.collectionTextQuery(); post.put("defType", "edismax"); for (String fq : solrFQ) post.add(CommonParams.FQ, fq); post.put(CommonParams.Q, solrQ.toString()); post.put(CommonParams.ROWS, post.remove("num")); post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)); // set ranking final Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(0); final String qf = ranking.getQueryFields(); if (!qf.isEmpty()) post.put(DisMaxParams.QF, qf); if (post.containsKey(CommonParams.SORT)) { // if a gsa-style sort attribute is given, use this to set the solr sort attribute GSAResponseWriter.Sort sort = new GSAResponseWriter.Sort(post.get(CommonParams.SORT, "")); String sorts = sort.toSolr(); if (sorts == null) { post.remove(CommonParams.SORT); } else { post.put(CommonParams.SORT, sorts); } } else { // if no such sort attribute is given, use the ranking as configured for YaCy String fq = ranking.getFilterQuery(); String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); if (fq.length() > 0) post.put(CommonParams.FQ, fq); if (bq.length() > 0) post.put(DisMaxParams.BQ, bq); if (bf.length() > 0) post.put("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } String daterange[] = post.remove("daterange"); if (daterange != null) { String origfq = post.get(CommonParams.FQ); String datefq = ""; for (String dr : daterange) { String from_to[] = dr.endsWith("..") ? new String[] { dr.substring(0, dr.length() - 2), "" } : dr.startsWith("..") ? new String[] { "", dr.substring(2) } : dr.split("\\.\\."); if (from_to.length != 2) continue; Date from = HeaderFramework.parseGSAFS(from_to[0]); if (from == null) from = new Date(0); Date to = HeaderFramework.parseGSAFS(from_to[1]); if (to == null) to = new Date(); to.setTime(to.getTime() + 24L * 60L * 60L * 1000L); // we add a day because the day is inclusive String z = CollectionSchema.last_modified.getSolrFieldName() + ":[" + ISO8601Formatter.FORMATTER.format(from) + " TO " + ISO8601Formatter.FORMATTER.format(to) + "]"; datefq = datefq.length() == 0 ? z : " OR " + z; } if (datefq.length() > 0) post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? datefq : "(" + origfq + ") AND (" + datefq + ")"); } post.put(CommonParams.FL, CollectionSchema.content_type.getSolrFieldName() + ',' + CollectionSchema.id.getSolrFieldName() + ',' + CollectionSchema.sku.getSolrFieldName() + ',' + CollectionSchema.title.getSolrFieldName() + ',' + CollectionSchema.description_txt.getSolrFieldName() + ',' + CollectionSchema.load_date_dt.getSolrFieldName() + ',' + CollectionSchema.last_modified.getSolrFieldName() + ',' + CollectionSchema.size_i.getSolrFieldName()); post.put("hl", "true"); post.put("hl.q", originalQuery); post.put("hl.fl", CollectionSchema.description_txt + "," + CollectionSchema.h4_txt.getSolrFieldName() + "," + CollectionSchema.h3_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()); post.put("hl.alternateField", CollectionSchema.description_txt.getSolrFieldName()); post.put("hl.simple.pre", "<b>"); post.put("hl.simple.post", "</b>"); post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)); //String[] access = post.remove("access"); //String[] entqr = post.remove("entqr"); // add sites operator String[] site = post.remove("site"); // example: col1|col2 if (site != null && site[0].length() > 0) { String origfq = post.get(CommonParams.FQ); String sitefq = QueryModifier.parseCollectionExpression(site[0]); post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? sitefq : "(" + origfq + ") AND (" + sitefq + ")"); } // get the embedded connector EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); if (connector == null) return; // do the solr request SolrQueryRequest req = connector.request(post.toSolrParams(null)); SolrQueryResponse response = null; Exception e = null; try { response = connector.query(req); } catch (final SolrException ee) { e = ee; } if (response != null) e = response.getException(); if (e != null) { ConcurrentLog.logException(e); if (req != null) req.close(); SolrRequestInfo.clearRequestInfo(); return; } // set some context for the writer /* Map<Object,Object> context = req.getContext(); context.put("ip", header.get("CLIENTIP", "")); context.put("client", "vsm_frontent"); context.put("sort", sort.sort); context.put("site", site == null ? "" : site); context.put("access", access == null ? "p" : access[0]); context.put("entqr", entqr == null ? "3" : entqr[0]); */ // write the result directly to the output stream Writer ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); try { responseWriter.write(ow, req, response); ow.flush(); } catch (final IOException e1) { } finally { req.close(); SolrRequestInfo.clearRequestInfo(); try { ow.close(); } catch (final IOException e1) { } } // log result Object rv = response.getValues().get("response"); int matches = 0; if (rv != null && rv instanceof ResultContext) { matches = ((ResultContext) rv).docs.matches(); } else if (rv != null && rv instanceof SolrDocumentList) { matches = (int) ((SolrDocumentList) rv).getNumFound(); } AccessTracker.addToDump(originalQuery, Integer.toString(matches)); ConcurrentLog.info("GSA Query", "results: " + matches + ", for query:" + post.toString()); }
From source file:net.yacy.http.servlets.SolrSelectServlet.java
License:Open Source License
@Override public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException { HttpServletRequest hrequest = (HttpServletRequest) request; HttpServletResponse hresponse = (HttpServletResponse) response; SolrQueryRequest req = null;//from w ww. j av a 2 s . c o m final Method reqMethod = Method.getMethod(hrequest.getMethod()); Writer out = null; try { // prepare request to solr MultiMapSolrParams mmsp = SolrRequestParsers.parseQueryString(hrequest.getQueryString()); Switchboard sb = Switchboard.getSwitchboard(); boolean authenticated = true; // count remote searches if this was part of a p2p search if (mmsp.getMap().containsKey("partitions")) { final int partitions = mmsp.getInt("partitions", 30); sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter } // get the ranking profile id int profileNr = mmsp.getInt("profileNr", 0); // rename post fields according to result style String querystring = ""; if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey(CommonParams.QUERY)) { querystring = mmsp.get(CommonParams.QUERY, ""); mmsp.getMap().remove(CommonParams.QUERY); QueryModifier modifier = new QueryModifier(0); querystring = modifier.parse(querystring); modifier.apply(mmsp); QueryGoal qg = new QueryGoal(querystring); StringBuilder solrQ = qg.collectionTextQuery(); mmsp.getMap().put(CommonParams.Q, new String[] { solrQ.toString() }); // sru patch } String q = mmsp.get(CommonParams.Q, ""); if (querystring.length() == 0) querystring = q; if (!mmsp.getMap().containsKey(CommonParams.START)) { int startRecord = mmsp.getFieldInt("startRecord", null, 0); mmsp.getMap().remove("startRecord"); mmsp.getMap().put(CommonParams.START, new String[] { Integer.toString(startRecord) }); // sru patch } if (!mmsp.getMap().containsKey(CommonParams.ROWS)) { int maximumRecords = mmsp.getFieldInt("maximumRecords", null, 10); mmsp.getMap().remove("maximumRecords"); mmsp.getMap().put(CommonParams.ROWS, new String[] { Integer.toString(maximumRecords) }); // sru patch } mmsp.getMap().put(CommonParams.ROWS, new String[] { Integer .toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)) }); // set ranking according to profile number if ranking attributes are not given in the request Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); if (!mmsp.getMap().containsKey(CommonParams.SORT) && !mmsp.getMap().containsKey(DisMaxParams.BQ) && !mmsp.getMap().containsKey(DisMaxParams.BF) && !mmsp.getMap().containsKey("boost")) { if (!mmsp.getMap().containsKey("defType")) mmsp.getMap().put("defType", new String[] { "edismax" }); String fq = ranking.getFilterQuery(); String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); if (fq.length() > 0) mmsp.getMap().put(CommonParams.FQ, new String[] { fq }); if (bq.length() > 0) mmsp.getMap().put(DisMaxParams.BQ, new String[] { bq }); if (bf.length() > 0) mmsp.getMap().put("boost", new String[] { bf }); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } // get a response writer for the result String wt = mmsp.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt); if (responseWriter == null) throw new ServletException("no response writer"); if (responseWriter instanceof OpensearchResponseWriter) { // set the title every time, it is possible that it has changed final String promoteSearchPageGreeting = (sb .getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? sb.getConfig("network.unit.description", "") : sb.getConfig(SwitchboardConstants.GREETING, ""); ((OpensearchResponseWriter) responseWriter).setTitle(promoteSearchPageGreeting); } // if this is a call to YaCys special search formats, enhance the query with field assignments if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(mmsp.get("hl", "true"))) { // add options for snippet generation if (!mmsp.getMap().containsKey("hl.q")) mmsp.getMap().put("hl.q", new String[] { q }); if (!mmsp.getMap().containsKey("hl.fl")) mmsp.getMap().put("hl.fl", new String[] { CollectionSchema.description_txt + "," + CollectionSchema.h4_txt.getSolrFieldName() + "," + CollectionSchema.h3_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName() }); if (!mmsp.getMap().containsKey("hl.alternateField")) mmsp.getMap().put("hl.alternateField", new String[] { CollectionSchema.description_txt.getSolrFieldName() }); if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[] { "<b>" }); if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[] { "</b>" }); if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[] { Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH) }); } // get the embedded connector String requestURI = hrequest.getRequestURI(); boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); mmsp.getMap().remove("core"); SolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); if (connector == null) { connector = defaultConnector ? sb.index.fulltext().getDefaultConnector() : sb.index.fulltext().getConnectorForRead(WebgraphSchema.CORE_NAME); } if (connector == null) throw new ServletException("no core"); // add default queryfield parameter according to local ranking config (or defaultfield) if (ranking != null) { // ranking normally never null final String qf = ranking.getQueryFields(); if (qf.length() > 4) { // make sure qf has content (else use df) addParam(DisMaxParams.QF, qf, mmsp.getMap()); // add QF that we set to be best suited for our index // TODO: if every peer applies a decent QF itself, this can be reverted to getMap().put() } else { mmsp.getMap().put(CommonParams.DF, new String[] { CollectionSchema.text_t.getSolrFieldName() }); } } else { mmsp.getMap().put(CommonParams.DF, new String[] { CollectionSchema.text_t.getSolrFieldName() }); } // do the solr request, generate facets if we use a special YaCy format final SolrQueryResponse rsp; if (connector instanceof EmbeddedSolrConnector) { req = ((EmbeddedSolrConnector) connector).request(mmsp); rsp = ((EmbeddedSolrConnector) connector).query(req); // prepare response hresponse.setHeader("Cache-Control", "no-cache, no-store"); HttpCacheHeaderUtil.checkHttpCachingVeto(rsp, hresponse, reqMethod); // check error if (rsp.getException() != null) { AccessTracker.addToDump(querystring, "0", new Date()); sendError(hresponse, rsp.getException()); return; } NamedList<?> values = rsp.getValues(); DocList r = ((ResultContext) values.get("response")).docs; int numFound = r.matches(); AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date()); // write response header final String contentType = responseWriter.getContentType(req, rsp); if (null != contentType) response.setContentType(contentType); if (Method.HEAD == reqMethod) { return; } // write response body if (responseWriter instanceof BinaryResponseWriter) { ((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp); } else { out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), UTF8.charset)); responseWriter.write(out, req, rsp); out.flush(); } } else { // write a 'faked' response using a call to the backend SolrDocumentList sdl = connector.getDocumentListByQuery(mmsp.getMap().get(CommonParams.Q)[0], mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0], Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]), Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]), mmsp.getMap().get(CommonParams.FL)); OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream()); EnhancedXMLResponseWriter.write(osw, req, sdl); osw.close(); } } catch (final Throwable ex) { sendError(hresponse, ex); } finally { if (req != null) { req.close(); } SolrRequestInfo.clearRequestInfo(); if (out != null) try { out.close(); } catch (final IOException e1) { } } }
From source file:net.yacy.search.AutoSearch.java
License:Open Source License
/** * Calls one peer for search results of the current query and adds it to the * local index. Depending on peers SolrAvailable flag the a solr query or * opensearch/rss query is used.// w ww .j av a2s. c om * * @param seed the peer to ask */ private void processSingleTarget(Seed seed) { ConcurrentLog.fine(AutoSearch.class.getName(), "ask " + seed.getIP() + " " + seed.getName() + " for query=" + currentQuery); if (seed.getFlagSolrAvailable()) { // do a solr query SolrDocumentList docList = null; SolrQuery solrQuery = new SolrQuery(); // use remote defaults and ranking (to query their index right) solrQuery.set(CommonParams.Q, currentQuery + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)"); // except this yacy special solrQuery.set("q.op", "AND"); // except ... no one word matches please solrQuery.set(CommonParams.ROWS, "20"); this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")"); try { RemoteInstance instance = new RemoteInstance( "http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection try { SolrConnector solrConnector = new RemoteSolrConnector(instance, true, null); if (!solrConnector.isClosed()) { try { QueryResponse rsp = solrConnector.getResponseByParams(solrQuery); docList = rsp.getResults(); } catch (Throwable e) { } finally { solrConnector.close(); } } } catch (Throwable ee) { } finally { instance.close(); } if (docList != null) { for (SolrDocument d : docList) { sb.index.fulltext() .putDocument(sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(d)); this.gotresults++; } ConcurrentLog.info(AutoSearch.class.getName(), "added " + docList.size() + " results from " + seed.getName() + " to index for solrquery=" + currentQuery); } } catch (Throwable eee) { } } else { // do a yacysearch.rss query final String rssSearchServiceURL = "http://" + seed.getPublicAddress(seed.getIP()) + "/yacysearch.rss"; try { RSSFeed feed = loadSRURSS(rssSearchServiceURL, currentQuery, 0, 20, CacheStrategy.IFFRESH, false, // just local, as we ask others too ClientIdentification.yacyInternetCrawlerAgent); final List<DigestURL> urls = new ArrayList<DigestURL>(); for (final MultiProtocolURL entry : feed.getLinks()) { urls.add(new DigestURL(entry, (byte[]) null)); this.gotresults++; } sb.addToIndex(urls, null, "AutoSearch", null, true); ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery); } catch (IOException ex) { ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName()); } } }
From source file:opennlp.tools.similarity.apps.solr.IterativeQueryComponent.java
License:Apache License
private ResponseBuilder substituteField(ResponseBuilder rb, String newFieldName) { SolrParams params = rb.req.getParams(); String query = params.get("q"); String currField = StringUtils.substringBetween(" " + query, " ", ":"); if (currField != null && newFieldName != null) query = query.replace(currField, newFieldName); NamedList values = params.toNamedList(); values.remove("q"); values.add("q", query); params = SolrParams.toSolrParams(values); rb.req.setParams(params);// www.ja va2s . c o m rb.setQueryString(query); String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE); // get it from the response builder to give a different component a chance // to set it. String queryString = rb.getQueryString(); if (queryString == null) { // this is the normal way it's set. queryString = params.get(CommonParams.Q); rb.setQueryString(queryString); } QParser parser = null; try { parser = QParser.getParser(rb.getQueryString(), defType, rb.req); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } Query q = null; try { q = parser.getQuery(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } if (q == null) { // normalize a null query to a query that matches nothing q = new BooleanQuery(); } rb.setQuery(q); try { rb.setSortSpec(parser.getSort(true)); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } rb.setQparser(parser); /* try { rb.setScoreDoc(parser.getPaging()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } */ String[] fqs = rb.req.getParams().getParams(CommonParams.FQ); if (fqs != null && fqs.length != 0) { List<Query> filters = rb.getFilters(); if (filters == null) { filters = new ArrayList<Query>(fqs.length); } for (String fq : fqs) { if (fq != null && fq.trim().length() != 0) { QParser fqp = null; try { fqp = QParser.getParser(fq, null, rb.req); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } try { filters.add(fqp.getQuery()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } } // only set the filters if they are not empty otherwise // fq=&someotherParam= will trigger all docs filter for every request // if filter cache is disabled if (!filters.isEmpty()) { rb.setFilters(filters); } } return rb; }
From source file:opennlp.tools.similarity.apps.solr.IterativeSearchRequestHandler.java
License:Apache License
public void handleRequestBody1(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // extract params from request SolrParams params = req.getParams(); String q = params.get(CommonParams.Q); String[] fqs = params.getParams(CommonParams.FQ); int start = 0; try {//from w w w.ja v a 2s .c o m start = Integer.parseInt(params.get(CommonParams.START)); } catch (Exception e) { /* default */ } int rows = 0; try { rows = Integer.parseInt(params.get(CommonParams.ROWS)); } catch (Exception e) { /* default */ } //SolrPluginUtils.setReturnFields(req, rsp); // build initial data structures SolrDocumentList results = new SolrDocumentList(); SolrIndexSearcher searcher = req.getSearcher(); Map<String, SchemaField> fields = req.getSchema().getFields(); int ndocs = start + rows; Filter filter = buildFilter(fqs, req); Set<Integer> alreadyFound = new HashSet<Integer>(); // invoke the various sub-handlers in turn and return results doSearch1(results, searcher, q, filter, ndocs, req, fields, alreadyFound); // ... more sub-handler calls here ... // build and write response float maxScore = 0.0F; int numFound = 0; List<SolrDocument> slice = new ArrayList<SolrDocument>(); for (Iterator<SolrDocument> it = results.iterator(); it.hasNext();) { SolrDocument sdoc = it.next(); Float score = (Float) sdoc.getFieldValue("score"); if (maxScore < score) { maxScore = score; } if (numFound >= start && numFound < start + rows) { slice.add(sdoc); } numFound++; } results.clear(); results.addAll(slice); results.setNumFound(numFound); results.setMaxScore(maxScore); results.setStart(start); rsp.add("response", results); }
From source file:org.alfresco.solr.component.AsyncBuildSuggestComponent.java
License:Open Source License
/** * Responsible for using the specified suggester to get the suggestions * for the query and write the results //w w w . j a v a2 s.c o m * */ @Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); LOG.debug("SuggestComponent process with : " + params); if (!params.getBool(COMPONENT_NAME, false) || suggesters.isEmpty()) { return; } boolean buildAll = params.getBool(SUGGEST_BUILD_ALL, false); boolean reloadAll = params.getBool(SUGGEST_RELOAD_ALL, false); Set<SolrSuggester> querySuggesters; try { querySuggesters = getSuggesters(params); } catch (IllegalArgumentException ex) { if (!buildAll && !reloadAll) { throw ex; } else { querySuggesters = new HashSet<>(); } } String query = params.get(SUGGEST_Q); if (query == null) { query = rb.getQueryString(); if (query == null) { query = params.get(CommonParams.Q); } } if (query != null) { int count = params.getInt(SUGGEST_COUNT, 1); boolean highlight = params.getBool(SUGGEST_HIGHLIGHT, false); boolean allTermsRequired = params.getBool(SUGGEST_ALL_TERMS_REQUIRED, true); String contextFilter = params.get(SUGGEST_CONTEXT_FILTER_QUERY); if (contextFilter != null) { contextFilter = contextFilter.trim(); if (contextFilter.length() == 0) { contextFilter = null; } } SuggesterOptions options = new SuggesterOptions(new CharsRef(query), count, contextFilter, allTermsRequired, highlight); Map<String, SimpleOrderedMap<NamedList<Object>>> namedListResults = new HashMap<>(); for (SolrSuggester suggester : querySuggesters) { SuggesterResult suggesterResult = suggester.getSuggestions(options); toNamedList(suggesterResult, namedListResults); } rb.rsp.add(SuggesterResultLabels.SUGGEST, namedListResults); } }