List of usage examples for org.apache.solr.common.params CommonParams ROWS
String ROWS
To view the source code for org.apache.solr.common.params CommonParams ROWS.
Click Source Link
From source file:net.yacy.cora.federate.solr.connector.MirrorSolrConnector.java
License:Open Source License
@Override public QueryResponse getResponseByParams(ModifiableSolrParams query) throws IOException, SolrException { Integer count0 = query.getInt(CommonParams.ROWS); int count = count0 == null ? 10 : count0.intValue(); Integer start0 = query.getInt(CommonParams.START); int start = start0 == null ? 0 : start0.intValue(); if (this.solr0 == null && this.solr1 == null) return new QueryResponse(); if (this.solr0 != null && this.solr1 == null) { QueryResponse list = this.solr0.getResponseByParams(query); return list; }/*from w w w . j av a 2 s . c o m*/ if (this.solr1 != null && this.solr0 == null) { QueryResponse list = this.solr1.getResponseByParams(query); return list; } // combine both lists QueryResponse rsp = this.solr0.getResponseByParams(query); final SolrDocumentList l = rsp.getResults(); if (l.size() >= count) return rsp; // at this point we need to know how many results are in solr0 // compute this with a very bad hack; replace with better method later int size0 = 0; { //bad hack - TODO: replace query.set(CommonParams.START, 0); query.set(CommonParams.ROWS, Integer.MAX_VALUE); QueryResponse lHack = this.solr0.getResponseByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); size0 = lHack.getResults().size(); } // now use the size of the first query to do a second query query.set(CommonParams.START, start + l.size() - size0); query.set(CommonParams.ROWS, count - l.size()); QueryResponse rsp1 = this.solr1.getResponseByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); // TODO: combine both return rsp1; }
From source file:net.yacy.cora.federate.solr.connector.MirrorSolrConnector.java
License:Open Source License
@Override public SolrDocumentList getDocumentListByParams(ModifiableSolrParams query) throws IOException, SolrException { Integer count0 = query.getInt(CommonParams.ROWS); int count = count0 == null ? 10 : count0.intValue(); Integer start0 = query.getInt(CommonParams.START); int start = start0 == null ? 0 : start0.intValue(); if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList(); if (this.solr0 != null && this.solr1 == null) { SolrDocumentList list = this.solr0.getDocumentListByParams(query); return list; }/* w ww .ja va 2 s . co m*/ if (this.solr1 != null && this.solr0 == null) { SolrDocumentList list = this.solr1.getDocumentListByParams(query); return list; } // combine both lists final SolrDocumentList l = this.solr0.getDocumentListByParams(query); if (l.size() >= count) return l; // at this point we need to know how many results are in solr0 // compute this with a very bad hack; replace with better method later int size0 = 0; { //bad hack - TODO: replace query.set(CommonParams.START, 0); query.set(CommonParams.ROWS, Integer.MAX_VALUE); final SolrDocumentList lHack = this.solr0.getDocumentListByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); size0 = lHack.size(); } // now use the size of the first query to do a second query query.set(CommonParams.START, start + l.size() - size0); query.set(CommonParams.ROWS, count - l.size()); final SolrDocumentList l1 = this.solr1.getDocumentListByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); // TODO: combine both return l1; }
From source file:net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter.java
License:Open Source License
@Override public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException { assert rsp.getValues().get("responseHeader") != null; assert rsp.getValues().get("response") != null; long start = System.currentTimeMillis(); SimpleOrderedMap<Object> responseHeader = (SimpleOrderedMap<Object>) rsp.getResponseHeader(); DocList response = ((ResultContext) rsp.getValues().get("response")).docs; @SuppressWarnings("unchecked") SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) rsp.getValues().get("highlighting"); Map<String, LinkedHashSet<String>> snippets = OpensearchResponseWriter.highlighting(highlighting); Map<Object, Object> context = request.getContext(); // parse response header ResHead resHead = new ResHead(); NamedList<?> val0 = (NamedList<?>) responseHeader.get("params"); resHead.rows = Integer.parseInt((String) val0.get(CommonParams.ROWS)); resHead.offset = response.offset(); // equal to 'start' resHead.numFound = response.matches(); //resHead.df = (String) val0.get("df"); //resHead.q = (String) val0.get("q"); //resHead.wt = (String) val0.get("wt"); //resHead.status = (Integer) responseHeader.get("status"); //resHead.QTime = (Integer) responseHeader.get("QTime"); //resHead.maxScore = response.maxScore(); // write header writer.write(XML_START);/* w ww.j a va 2s . c om*/ String query = request.getParams().get("originalQuery"); String site = getContextString(context, "site", ""); String sort = getContextString(context, "sort", ""); String client = getContextString(context, "client", ""); String ip = getContextString(context, "ip", ""); String access = getContextString(context, "access", ""); String entqr = getContextString(context, "entqr", ""); OpensearchResponseWriter.solitaireTag(writer, "TM", Long.toString(System.currentTimeMillis() - start)); OpensearchResponseWriter.solitaireTag(writer, "Q", query); paramTag(writer, "sort", sort); paramTag(writer, "output", "xml_no_dtd"); paramTag(writer, "ie", "UTF-8"); paramTag(writer, "oe", "UTF-8"); paramTag(writer, "client", client); paramTag(writer, "q", query); paramTag(writer, "site", site); paramTag(writer, "start", Integer.toString(resHead.offset)); paramTag(writer, "num", Integer.toString(resHead.rows)); paramTag(writer, "ip", ip); paramTag(writer, "access", access); // p - search only public content, s - search only secure content, a - search all content, both public and secure paramTag(writer, "entqr", entqr); // query expansion policy; (entqr=1) -- Uses only the search appliance's synonym file, (entqr=1) -- Uses only the search appliance's synonym file, (entqr=3) -- Uses both standard and local synonym files. // body introduction final int responseCount = response.size(); writer.write("<RES SN=\"" + (resHead.offset + 1) + "\" EN=\"" + (resHead.offset + responseCount) + "\">"); writer.write(lb); // The index (1-based) of the first and last search result returned in this result set. writer.write("<M>" + resHead.numFound + "</M>"); writer.write(lb); // The estimated total number of results for the search. writer.write("<FI/>"); writer.write(lb); // Indicates that document filtering was performed during this search. int nextStart = resHead.offset + responseCount; int nextNum = Math.min(resHead.numFound - nextStart, responseCount < resHead.rows ? 0 : resHead.rows); int prevStart = resHead.offset - resHead.rows; if (prevStart >= 0 || nextNum > 0) { writer.write("<NB>"); if (prevStart >= 0) { writer.write("<PU>"); XML.escapeCharData("/gsa/search?q=" + request.getParams().get(CommonParams.Q) + "&site=" + site + "&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access + "&sort=" + sort + "&start=" + prevStart + "&sa=N", writer); // a relative URL pointing to the NEXT results page. writer.write("</PU>"); } if (nextNum > 0) { writer.write("<NU>"); XML.escapeCharData("/gsa/search?q=" + request.getParams().get(CommonParams.Q) + "&site=" + site + "&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access + "&sort=" + sort + "&start=" + nextStart + "&num=" + nextNum + "&sa=N", writer); // a relative URL pointing to the NEXT results page. writer.write("</NU>"); } writer.write("</NB>"); } writer.write(lb); // parse body SolrIndexSearcher searcher = request.getSearcher(); DocIterator iterator = response.iterator(); String urlhash = null; for (int i = 0; i < responseCount; i++) { int id = iterator.nextDoc(); Document doc = searcher.doc(id, SOLR_FIELDS); List<IndexableField> fields = doc.getFields(); // pre-scan the fields to get the mime-type String mime = ""; for (IndexableField value : fields) { String fieldName = value.name(); if (CollectionSchema.content_type.getSolrFieldName().equals(fieldName)) { mime = value.stringValue(); break; } } // write the R header for a search result writer.write("<R N=\"" + (resHead.offset + i + 1) + "\"" + (i == 1 ? " L=\"2\"" : "") + (mime != null && mime.length() > 0 ? " MIME=\"" + mime + "\"" : "") + ">"); writer.write(lb); //List<String> texts = new ArrayList<String>(); List<String> descriptions = new ArrayList<String>(); List<String> collections = new ArrayList<String>(); int size = 0; boolean title_written = false; // the solr index may contain several; we take only the first which should be the visible tag in <title></title> String title = null; for (IndexableField value : fields) { String fieldName = value.name(); // apply generic matching rule String stag = field2tag.get(fieldName); if (stag != null) { OpensearchResponseWriter.solitaireTag(writer, stag, value.stringValue()); continue; } // if the rule is not generic, use the specific here if (CollectionSchema.id.getSolrFieldName().equals(fieldName)) { urlhash = value.stringValue(); continue; } if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { OpensearchResponseWriter.solitaireTag(writer, GSAToken.U.name(), value.stringValue()); OpensearchResponseWriter.solitaireTag(writer, GSAToken.UE.name(), value.stringValue()); continue; } if (CollectionSchema.title.getSolrFieldName().equals(fieldName) && !title_written) { title = value.stringValue(); OpensearchResponseWriter.solitaireTag(writer, GSAToken.T.name(), highlight(title, query)); //texts.add(value.stringValue()); title_written = true; continue; } if (CollectionSchema.description_txt.getSolrFieldName().equals(fieldName)) { descriptions.add(value.stringValue()); //texts.adds(description); continue; } if (CollectionSchema.last_modified.getSolrFieldName().equals(fieldName)) { Date d = new Date(Long.parseLong(value.stringValue())); writer.write("<FS NAME=\"date\" VALUE=\"" + HeaderFramework.formatGSAFS(d) + "\"/>\n"); //OpensearchResponseWriter.solitaireTag(writer, GSAToken.CACHE_LAST_MODIFIED.getSolrFieldName(), HeaderFramework.formatRFC1123(d)); //texts.add(value.stringValue()); continue; } if (CollectionSchema.load_date_dt.getSolrFieldName().equals(fieldName)) { Date d = new Date(Long.parseLong(value.stringValue())); OpensearchResponseWriter.solitaireTag(writer, GSAToken.CRAWLDATE.name(), HeaderFramework.formatRFC1123(d)); //texts.add(value.stringValue()); continue; } if (CollectionSchema.size_i.getSolrFieldName().equals(fieldName)) { size = value.stringValue() != null && value.stringValue().length() > 0 ? Integer.parseInt(value.stringValue()) : -1; continue; } if (CollectionSchema.collection_sxt.getSolrFieldName().equals(fieldName)) { collections.add(value.stringValue()); continue; } //System.out.println("superfluous field: " + fieldName + ": " + value.stringValue()); // this can be avoided setting the enableLazyFieldLoading = false in solrconfig.xml } // compute snippet from texts LinkedHashSet<String> snippet = urlhash == null ? null : snippets.get(urlhash); OpensearchResponseWriter.removeSubsumedTitle(snippet, title); OpensearchResponseWriter.solitaireTag(writer, GSAToken.S.name(), snippet == null || snippet.size() == 0 ? (descriptions.size() > 0 ? descriptions.get(0) : "") : OpensearchResponseWriter.getLargestSnippet(snippet)); OpensearchResponseWriter.solitaireTag(writer, GSAToken.GD.name(), descriptions.size() > 0 ? descriptions.get(0) : ""); String cols = collections.toString(); if (collections.size() > 0) OpensearchResponseWriter.solitaireTag(writer, "COLS" /*SPECIAL!*/, collections.size() > 1 ? cols.substring(1, cols.length() - 1).replaceAll(" ", "") : collections.get(0)); writer.write("<HAS><L/><C SZ=\""); writer.write(Integer.toString(size / 1024)); writer.write("k\" CID=\""); writer.write(urlhash); writer.write("\" ENC=\"UTF-8\"/></HAS>\n"); if (YaCyVer == null) YaCyVer = yacyVersion.thisVersion().getName() + "/" + Switchboard.getSwitchboard().peers.mySeed().hash; OpensearchResponseWriter.solitaireTag(writer, GSAToken.ENT_SOURCE.name(), YaCyVer); OpensearchResponseWriter.closeTag(writer, "R"); } writer.write("</RES>"); writer.write(lb); writer.write(XML_STOP); }
From source file:net.yacy.cora.federate.SolrFederateSearchConnector.java
License:Open Source License
/** * Core query implementation/*from w w w. ja va2 s . com*/ * all query and search routines will use this routine to query the remote system * * @param query * @return list of solr documents (metadata) accordng to local YaCy internal schema */ @Override public List<URIMetadataNode> query(QueryParams query) { List<URIMetadataNode> docs = new ArrayList<URIMetadataNode>(); Collection<String> remotecorename = new ArrayList<String>(); remotecorename.add(corename); ModifiableSolrParams msp = new SolrQuery(query.getQueryGoal().getQueryString(false)); msp.add(CommonParams.QT, "/"); // important to override default append of /select msp.add(CommonParams.ROWS, Integer.toString(query.itemsPerPage)); try { RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000); try { SolrConnector solrConnector = new RemoteSolrConnector(instance, false, null); try { this.lastaccesstime = System.currentTimeMillis(); SolrDocumentList docList = solrConnector.getDocumentListByParams(msp); // convert to YaCy schema documentlist for (SolrDocument doc : docList) { URIMetadataNode anew = toYaCySchema(doc); docs.add(anew); } } catch (IOException | SolrException e) { } finally { solrConnector.close(); } } catch (Throwable ee) { } finally { instance.close(); } } catch (IOException eee) { } return docs; }
From source file:net.yacy.http.servlets.GSAsearchServlet.java
License:Open Source License
/** * from here copy of htroot/gsa/gsasearchresult.java * with modification to use HttpServletRequest instead of (yacy) RequestHeader *//*from w w w .ja va 2 s. c o m*/ public static void respond(final HttpServletRequest header, final Switchboard sb, final OutputStream out) { // remember the peer contact for peer statistics String clientip = header.getHeader(HeaderFramework.CONNECTION_PROP_CLIENTIP); if (clientip == null) clientip = "<unknown>"; // read an artificial header addendum String userAgent = header.getHeader(HeaderFramework.USER_AGENT); if (userAgent == null) userAgent = "<unknown>"; sb.peers.peerActions.setUserAgent(clientip, userAgent); // --- handled by Servlet securityHandler // check if user is allowed to search (can be switched in /ConfigPortal.html) boolean authenticated = header.isUserInRole(UserDB.AccessRight.ADMIN_RIGHT.toString()); //sb.adminAuthenticated(header) >= 2; // final boolean searchAllowed = authenticated || sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true); // if (!searchAllowed) return null; // create post serverObjects post = new serverObjects(); post.put(CommonParams.Q, ""); post.put("num", "0"); // convert servletrequest parameter to old style serverObjects map Map<String, String[]> map = header.getParameterMap(); Iterator<Map.Entry<String, String[]>> it = map.entrySet().iterator(); while (it.hasNext()) { Map.Entry<String, String[]> param = it.next(); post.put(param.getKey(), param.getValue()); // hint: post.put uses String[] for String value anyways } ConcurrentLog.info("GSA Query", post.toString()); sb.intermissionAllThreads(3000); // tell all threads to do nothing for a specific time // rename post fields according to result style //post.put(CommonParams.Q, post.remove("q")); // same as solr //post.put(CommonParams.START, post.remove("start")); // same as solr //post.put(, post.remove("client"));//required, example: myfrontend //post.put(, post.remove("output"));//required, example: xml,xml_no_dtd String originalQuery = post.get(CommonParams.Q, ""); post.put("originalQuery", originalQuery); // get a solr query string QueryGoal qg = new QueryGoal(originalQuery); List<String> solrFQ = qg.collectionTextFilterQuery(false); StringBuilder solrQ = qg.collectionTextQuery(); post.put("defType", "edismax"); for (String fq : solrFQ) post.add(CommonParams.FQ, fq); post.put(CommonParams.Q, solrQ.toString()); post.put(CommonParams.ROWS, post.remove("num")); post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)); // set ranking final Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(0); final String qf = ranking.getQueryFields(); if (!qf.isEmpty()) post.put(DisMaxParams.QF, qf); if (post.containsKey(CommonParams.SORT)) { // if a gsa-style sort attribute is given, use this to set the solr sort attribute GSAResponseWriter.Sort sort = new GSAResponseWriter.Sort(post.get(CommonParams.SORT, "")); String sorts = sort.toSolr(); if (sorts == null) { post.remove(CommonParams.SORT); } else { post.put(CommonParams.SORT, sorts); } } else { // if no such sort attribute is given, use the ranking as configured for YaCy String fq = ranking.getFilterQuery(); String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); if (fq.length() > 0) post.put(CommonParams.FQ, fq); if (bq.length() > 0) post.put(DisMaxParams.BQ, bq); if (bf.length() > 0) post.put("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } String daterange[] = post.remove("daterange"); if (daterange != null) { String origfq = post.get(CommonParams.FQ); String datefq = ""; for (String dr : daterange) { String from_to[] = dr.endsWith("..") ? new String[] { dr.substring(0, dr.length() - 2), "" } : dr.startsWith("..") ? new String[] { "", dr.substring(2) } : dr.split("\\.\\."); if (from_to.length != 2) continue; Date from = HeaderFramework.parseGSAFS(from_to[0]); if (from == null) from = new Date(0); Date to = HeaderFramework.parseGSAFS(from_to[1]); if (to == null) to = new Date(); to.setTime(to.getTime() + 24L * 60L * 60L * 1000L); // we add a day because the day is inclusive String z = CollectionSchema.last_modified.getSolrFieldName() + ":[" + ISO8601Formatter.FORMATTER.format(from) + " TO " + ISO8601Formatter.FORMATTER.format(to) + "]"; datefq = datefq.length() == 0 ? z : " OR " + z; } if (datefq.length() > 0) post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? datefq : "(" + origfq + ") AND (" + datefq + ")"); } post.put(CommonParams.FL, CollectionSchema.content_type.getSolrFieldName() + ',' + CollectionSchema.id.getSolrFieldName() + ',' + CollectionSchema.sku.getSolrFieldName() + ',' + CollectionSchema.title.getSolrFieldName() + ',' + CollectionSchema.description_txt.getSolrFieldName() + ',' + CollectionSchema.load_date_dt.getSolrFieldName() + ',' + CollectionSchema.last_modified.getSolrFieldName() + ',' + CollectionSchema.size_i.getSolrFieldName()); post.put("hl", "true"); post.put("hl.q", originalQuery); post.put("hl.fl", CollectionSchema.description_txt + "," + CollectionSchema.h4_txt.getSolrFieldName() + "," + CollectionSchema.h3_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName()); post.put("hl.alternateField", CollectionSchema.description_txt.getSolrFieldName()); post.put("hl.simple.pre", "<b>"); post.put("hl.simple.post", "</b>"); post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH)); //String[] access = post.remove("access"); //String[] entqr = post.remove("entqr"); // add sites operator String[] site = post.remove("site"); // example: col1|col2 if (site != null && site[0].length() > 0) { String origfq = post.get(CommonParams.FQ); String sitefq = QueryModifier.parseCollectionExpression(site[0]); post.put(CommonParams.FQ, origfq == null || origfq.length() == 0 ? sitefq : "(" + origfq + ") AND (" + sitefq + ")"); } // get the embedded connector EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); if (connector == null) return; // do the solr request SolrQueryRequest req = connector.request(post.toSolrParams(null)); SolrQueryResponse response = null; Exception e = null; try { response = connector.query(req); } catch (final SolrException ee) { e = ee; } if (response != null) e = response.getException(); if (e != null) { ConcurrentLog.logException(e); if (req != null) req.close(); SolrRequestInfo.clearRequestInfo(); return; } // set some context for the writer /* Map<Object,Object> context = req.getContext(); context.put("ip", header.get("CLIENTIP", "")); context.put("client", "vsm_frontent"); context.put("sort", sort.sort); context.put("site", site == null ? "" : site); context.put("access", access == null ? "p" : access[0]); context.put("entqr", entqr == null ? "3" : entqr[0]); */ // write the result directly to the output stream Writer ow = new FastWriter(new OutputStreamWriter(out, UTF8.charset)); try { responseWriter.write(ow, req, response); ow.flush(); } catch (final IOException e1) { } finally { req.close(); SolrRequestInfo.clearRequestInfo(); try { ow.close(); } catch (final IOException e1) { } } // log result Object rv = response.getValues().get("response"); int matches = 0; if (rv != null && rv instanceof ResultContext) { matches = ((ResultContext) rv).docs.matches(); } else if (rv != null && rv instanceof SolrDocumentList) { matches = (int) ((SolrDocumentList) rv).getNumFound(); } AccessTracker.addToDump(originalQuery, Integer.toString(matches)); ConcurrentLog.info("GSA Query", "results: " + matches + ", for query:" + post.toString()); }
From source file:net.yacy.http.servlets.SolrSelectServlet.java
License:Open Source License
@Override public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException { HttpServletRequest hrequest = (HttpServletRequest) request; HttpServletResponse hresponse = (HttpServletResponse) response; SolrQueryRequest req = null;/* w w w .j ava 2 s. c om*/ final Method reqMethod = Method.getMethod(hrequest.getMethod()); Writer out = null; try { // prepare request to solr MultiMapSolrParams mmsp = SolrRequestParsers.parseQueryString(hrequest.getQueryString()); Switchboard sb = Switchboard.getSwitchboard(); boolean authenticated = true; // count remote searches if this was part of a p2p search if (mmsp.getMap().containsKey("partitions")) { final int partitions = mmsp.getInt("partitions", 30); sb.searchQueriesGlobal += 1.0f / partitions; // increase query counter } // get the ranking profile id int profileNr = mmsp.getInt("profileNr", 0); // rename post fields according to result style String querystring = ""; if (!mmsp.getMap().containsKey(CommonParams.Q) && mmsp.getMap().containsKey(CommonParams.QUERY)) { querystring = mmsp.get(CommonParams.QUERY, ""); mmsp.getMap().remove(CommonParams.QUERY); QueryModifier modifier = new QueryModifier(0); querystring = modifier.parse(querystring); modifier.apply(mmsp); QueryGoal qg = new QueryGoal(querystring); StringBuilder solrQ = qg.collectionTextQuery(); mmsp.getMap().put(CommonParams.Q, new String[] { solrQ.toString() }); // sru patch } String q = mmsp.get(CommonParams.Q, ""); if (querystring.length() == 0) querystring = q; if (!mmsp.getMap().containsKey(CommonParams.START)) { int startRecord = mmsp.getFieldInt("startRecord", null, 0); mmsp.getMap().remove("startRecord"); mmsp.getMap().put(CommonParams.START, new String[] { Integer.toString(startRecord) }); // sru patch } if (!mmsp.getMap().containsKey(CommonParams.ROWS)) { int maximumRecords = mmsp.getFieldInt("maximumRecords", null, 10); mmsp.getMap().remove("maximumRecords"); mmsp.getMap().put(CommonParams.ROWS, new String[] { Integer.toString(maximumRecords) }); // sru patch } mmsp.getMap().put(CommonParams.ROWS, new String[] { Integer .toString(Math.min(mmsp.getInt(CommonParams.ROWS, 10), (authenticated) ? 100000000 : 100)) }); // set ranking according to profile number if ranking attributes are not given in the request Ranking ranking = sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr); if (!mmsp.getMap().containsKey(CommonParams.SORT) && !mmsp.getMap().containsKey(DisMaxParams.BQ) && !mmsp.getMap().containsKey(DisMaxParams.BF) && !mmsp.getMap().containsKey("boost")) { if (!mmsp.getMap().containsKey("defType")) mmsp.getMap().put("defType", new String[] { "edismax" }); String fq = ranking.getFilterQuery(); String bq = ranking.getBoostQuery(); String bf = ranking.getBoostFunction(); if (fq.length() > 0) mmsp.getMap().put(CommonParams.FQ, new String[] { fq }); if (bq.length() > 0) mmsp.getMap().put(DisMaxParams.BQ, new String[] { bq }); if (bf.length() > 0) mmsp.getMap().put("boost", new String[] { bf }); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 } // get a response writer for the result String wt = mmsp.get(CommonParams.WT, "xml"); // maybe use /solr/select?q=*:*&start=0&rows=10&wt=exml QueryResponseWriter responseWriter = RESPONSE_WRITER.get(wt); if (responseWriter == null) throw new ServletException("no response writer"); if (responseWriter instanceof OpensearchResponseWriter) { // set the title every time, it is possible that it has changed final String promoteSearchPageGreeting = (sb .getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? sb.getConfig("network.unit.description", "") : sb.getConfig(SwitchboardConstants.GREETING, ""); ((OpensearchResponseWriter) responseWriter).setTitle(promoteSearchPageGreeting); } // if this is a call to YaCys special search formats, enhance the query with field assignments if ((responseWriter instanceof YJsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(mmsp.get("hl", "true"))) { // add options for snippet generation if (!mmsp.getMap().containsKey("hl.q")) mmsp.getMap().put("hl.q", new String[] { q }); if (!mmsp.getMap().containsKey("hl.fl")) mmsp.getMap().put("hl.fl", new String[] { CollectionSchema.description_txt + "," + CollectionSchema.h4_txt.getSolrFieldName() + "," + CollectionSchema.h3_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName() }); if (!mmsp.getMap().containsKey("hl.alternateField")) mmsp.getMap().put("hl.alternateField", new String[] { CollectionSchema.description_txt.getSolrFieldName() }); if (!mmsp.getMap().containsKey("hl.simple.pre")) mmsp.getMap().put("hl.simple.pre", new String[] { "<b>" }); if (!mmsp.getMap().containsKey("hl.simple.post")) mmsp.getMap().put("hl.simple.post", new String[] { "</b>" }); if (!mmsp.getMap().containsKey("hl.fragsize")) mmsp.getMap().put("hl.fragsize", new String[] { Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH) }); } // get the embedded connector String requestURI = hrequest.getRequestURI(); boolean defaultConnector = (requestURI.startsWith("/solr/" + WebgraphSchema.CORE_NAME)) ? false : requestURI.startsWith("/solr/" + CollectionSchema.CORE_NAME) || mmsp.get("core", CollectionSchema.CORE_NAME).equals(CollectionSchema.CORE_NAME); mmsp.getMap().remove("core"); SolrConnector connector = defaultConnector ? sb.index.fulltext().getDefaultEmbeddedConnector() : sb.index.fulltext().getEmbeddedConnector(WebgraphSchema.CORE_NAME); if (connector == null) { connector = defaultConnector ? sb.index.fulltext().getDefaultConnector() : sb.index.fulltext().getConnectorForRead(WebgraphSchema.CORE_NAME); } if (connector == null) throw new ServletException("no core"); // add default queryfield parameter according to local ranking config (or defaultfield) if (ranking != null) { // ranking normally never null final String qf = ranking.getQueryFields(); if (qf.length() > 4) { // make sure qf has content (else use df) addParam(DisMaxParams.QF, qf, mmsp.getMap()); // add QF that we set to be best suited for our index // TODO: if every peer applies a decent QF itself, this can be reverted to getMap().put() } else { mmsp.getMap().put(CommonParams.DF, new String[] { CollectionSchema.text_t.getSolrFieldName() }); } } else { mmsp.getMap().put(CommonParams.DF, new String[] { CollectionSchema.text_t.getSolrFieldName() }); } // do the solr request, generate facets if we use a special YaCy format final SolrQueryResponse rsp; if (connector instanceof EmbeddedSolrConnector) { req = ((EmbeddedSolrConnector) connector).request(mmsp); rsp = ((EmbeddedSolrConnector) connector).query(req); // prepare response hresponse.setHeader("Cache-Control", "no-cache, no-store"); HttpCacheHeaderUtil.checkHttpCachingVeto(rsp, hresponse, reqMethod); // check error if (rsp.getException() != null) { AccessTracker.addToDump(querystring, "0", new Date()); sendError(hresponse, rsp.getException()); return; } NamedList<?> values = rsp.getValues(); DocList r = ((ResultContext) values.get("response")).docs; int numFound = r.matches(); AccessTracker.addToDump(querystring, Integer.toString(numFound), new Date()); // write response header final String contentType = responseWriter.getContentType(req, rsp); if (null != contentType) response.setContentType(contentType); if (Method.HEAD == reqMethod) { return; } // write response body if (responseWriter instanceof BinaryResponseWriter) { ((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp); } else { out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), UTF8.charset)); responseWriter.write(out, req, rsp); out.flush(); } } else { // write a 'faked' response using a call to the backend SolrDocumentList sdl = connector.getDocumentListByQuery(mmsp.getMap().get(CommonParams.Q)[0], mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0], Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]), Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]), mmsp.getMap().get(CommonParams.FL)); OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream()); EnhancedXMLResponseWriter.write(osw, req, sdl); osw.close(); } } catch (final Throwable ex) { sendError(hresponse, ex); } finally { if (req != null) { req.close(); } SolrRequestInfo.clearRequestInfo(); if (out != null) try { out.close(); } catch (final IOException e1) { } } }
From source file:net.yacy.search.AutoSearch.java
License:Open Source License
/** * Calls one peer for search results of the current query and adds it to the * local index. Depending on peers SolrAvailable flag the a solr query or * opensearch/rss query is used.//from w ww. j av a 2 s. co m * * @param seed the peer to ask */ private void processSingleTarget(Seed seed) { ConcurrentLog.fine(AutoSearch.class.getName(), "ask " + seed.getIP() + " " + seed.getName() + " for query=" + currentQuery); if (seed.getFlagSolrAvailable()) { // do a solr query SolrDocumentList docList = null; SolrQuery solrQuery = new SolrQuery(); // use remote defaults and ranking (to query their index right) solrQuery.set(CommonParams.Q, currentQuery + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)"); // except this yacy special solrQuery.set("q.op", "AND"); // except ... no one word matches please solrQuery.set(CommonParams.ROWS, "20"); this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")"); try { RemoteInstance instance = new RemoteInstance( "http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection try { SolrConnector solrConnector = new RemoteSolrConnector(instance, true, null); if (!solrConnector.isClosed()) { try { QueryResponse rsp = solrConnector.getResponseByParams(solrQuery); docList = rsp.getResults(); } catch (Throwable e) { } finally { solrConnector.close(); } } } catch (Throwable ee) { } finally { instance.close(); } if (docList != null) { for (SolrDocument d : docList) { sb.index.fulltext() .putDocument(sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(d)); this.gotresults++; } ConcurrentLog.info(AutoSearch.class.getName(), "added " + docList.size() + " results from " + seed.getName() + " to index for solrquery=" + currentQuery); } } catch (Throwable eee) { } } else { // do a yacysearch.rss query final String rssSearchServiceURL = "http://" + seed.getPublicAddress(seed.getIP()) + "/yacysearch.rss"; try { RSSFeed feed = loadSRURSS(rssSearchServiceURL, currentQuery, 0, 20, CacheStrategy.IFFRESH, false, // just local, as we ask others too ClientIdentification.yacyInternetCrawlerAgent); final List<DigestURL> urls = new ArrayList<DigestURL>(); for (final MultiProtocolURL entry : feed.getLinks()) { urls.add(new DigestURL(entry, (byte[]) null)); this.gotresults++; } sb.addToIndex(urls, null, "AutoSearch", null, true); ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery); } catch (IOException ex) { ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName()); } } }
From source file:net.yacy.server.serverObjects.java
License:Open Source License
public MultiMapSolrParams toSolrParams(CollectionSchema[] facets) { // check if all required post fields are there if (!this.containsKey(CommonParams.DF)) this.put(CommonParams.DF, CollectionSchema.text_t.getSolrFieldName()); // set default field to the text field if (!this.containsKey(CommonParams.START)) this.put(CommonParams.START, "0"); // set default start item if (!this.containsKey(CommonParams.ROWS)) this.put(CommonParams.ROWS, "10"); // set default number of search results if (facets != null && facets.length > 0) { this.remove("facet"); this.put("facet", "true"); for (int i = 0; i < facets.length; i++) this.add("facet.field", facets[i].getSolrFieldName()); }//from www.j av a 2 s . c om return this.map; }
From source file:opennlp.tools.similarity.apps.solr.IterativeSearchRequestHandler.java
License:Apache License
public void handleRequestBody1(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // extract params from request SolrParams params = req.getParams(); String q = params.get(CommonParams.Q); String[] fqs = params.getParams(CommonParams.FQ); int start = 0; try {/*from ww w . j av a2 s. co m*/ start = Integer.parseInt(params.get(CommonParams.START)); } catch (Exception e) { /* default */ } int rows = 0; try { rows = Integer.parseInt(params.get(CommonParams.ROWS)); } catch (Exception e) { /* default */ } //SolrPluginUtils.setReturnFields(req, rsp); // build initial data structures SolrDocumentList results = new SolrDocumentList(); SolrIndexSearcher searcher = req.getSearcher(); Map<String, SchemaField> fields = req.getSchema().getFields(); int ndocs = start + rows; Filter filter = buildFilter(fqs, req); Set<Integer> alreadyFound = new HashSet<Integer>(); // invoke the various sub-handlers in turn and return results doSearch1(results, searcher, q, filter, ndocs, req, fields, alreadyFound); // ... more sub-handler calls here ... // build and write response float maxScore = 0.0F; int numFound = 0; List<SolrDocument> slice = new ArrayList<SolrDocument>(); for (Iterator<SolrDocument> it = results.iterator(); it.hasNext();) { SolrDocument sdoc = it.next(); Float score = (Float) sdoc.getFieldValue("score"); if (maxScore < score) { maxScore = score; } if (numFound >= start && numFound < start + rows) { slice.add(sdoc); } numFound++; } results.clear(); results.addAll(slice); results.setNumFound(numFound); results.setMaxScore(maxScore); results.setStart(start); rsp.add("response", results); }
From source file:org.alfresco.solr.component.spellcheck.AlfrescoSpellCheckCollator.java
License:Open Source License
public List<AlfrescoSpellCheckCollation> collate(SpellingResult result, String originalQuery, ResponseBuilder ultimateResponse) { List<AlfrescoSpellCheckCollation> collations = new ArrayList<>(); QueryComponent queryComponent = null; if (ultimateResponse.components != null) { for (SearchComponent sc : ultimateResponse.components) { if (sc instanceof QueryComponent) { queryComponent = (QueryComponent) sc; break; }/*from www . j a v a 2s . co m*/ } } boolean verifyCandidateWithQuery = true; int maxTries = maxCollationTries; int maxNumberToIterate = maxTries; if (maxTries < 1) { maxTries = 1; maxNumberToIterate = maxCollations; verifyCandidateWithQuery = false; } if (queryComponent == null && verifyCandidateWithQuery) { LOG.info( "Could not find an instance of QueryComponent. Disabling collation verification against the index."); maxTries = 1; verifyCandidateWithQuery = false; } docCollectionLimit = docCollectionLimit > 0 ? docCollectionLimit : 0; int maxDocId = -1; if (verifyCandidateWithQuery && docCollectionLimit > 0) { IndexReader reader = ultimateResponse.req.getSearcher().getIndexReader(); maxDocId = reader.maxDoc(); } JSONObject alfrescoJSON = (JSONObject) ultimateResponse.req.getContext().get(AbstractQParser.ALFRESCO_JSON); String originalAftsQuery = alfrescoJSON != null ? alfrescoJSON.getString("query") : ultimateResponse.getQueryString(); int tryNo = 0; int collNo = 0; PossibilityIterator possibilityIter = new PossibilityIterator(result.getSuggestions(), maxNumberToIterate, maxCollationEvaluations, suggestionsMayOverlap); while (tryNo < maxTries && collNo < maxCollations && possibilityIter.hasNext()) { PossibilityIterator.RankedSpellPossibility possibility = possibilityIter.next(); String collationQueryStr = getCollation(originalQuery, possibility.corrections); int hits = 0; String aftsQuery = null; if (verifyCandidateWithQuery) { tryNo++; SolrQueryRequest req = ultimateResponse.req; SolrParams origParams = req.getParams(); ModifiableSolrParams params = new ModifiableSolrParams(origParams); Iterator<String> origParamIterator = origParams.getParameterNamesIterator(); int pl = SpellingParams.SPELLCHECK_COLLATE_PARAM_OVERRIDE.length(); while (origParamIterator.hasNext()) { String origParamName = origParamIterator.next(); if (origParamName.startsWith(SpellingParams.SPELLCHECK_COLLATE_PARAM_OVERRIDE) && origParamName.length() > pl) { String[] val = origParams.getParams(origParamName); if (val.length == 1 && val[0].length() == 0) { params.set(origParamName.substring(pl), (String[]) null); } else { params.set(origParamName.substring(pl), val); } } } // we don't set the 'q' param, as we'll pass the query via JSON. // params.set(CommonParams.Q, collationQueryStr); params.remove(CommonParams.START); params.set(CommonParams.ROWS, "" + docCollectionLimit); // we don't want any stored fields params.set(CommonParams.FL, "id"); // we'll sort by doc id to ensure no scoring is done. params.set(CommonParams.SORT, "_docid_ asc"); // If a dismax query, don't add unnecessary clauses for scoring params.remove(DisMaxParams.TIE); params.remove(DisMaxParams.PF); params.remove(DisMaxParams.PF2); params.remove(DisMaxParams.PF3); params.remove(DisMaxParams.BQ); params.remove(DisMaxParams.BF); // Collate testing does not support Grouping (see SOLR-2577) params.remove(GroupParams.GROUP); boolean useQStr = true; if (alfrescoJSON != null) { try { aftsQuery = originalAftsQuery.replaceAll(Pattern.quote(originalQuery), Matcher.quoteReplacement(collationQueryStr)); alfrescoJSON.put("query", aftsQuery); req.getContext().put(AbstractQParser.ALFRESCO_JSON, alfrescoJSON); useQStr = false; } catch (JSONException e) { LOG.warn("Exception trying to get/set the query from/to ALFRESCO_JSON.]" + e); } } else { aftsQuery = collationQueryStr; } req.setParams(params); // creating a request here... make sure to close it! ResponseBuilder checkResponse = new ResponseBuilder(req, new SolrQueryResponse(), Arrays.<SearchComponent>asList(queryComponent)); checkResponse.setQparser(ultimateResponse.getQparser()); checkResponse.setFilters(ultimateResponse.getFilters()); checkResponse.components = Arrays.<SearchComponent>asList(queryComponent); if (useQStr) { checkResponse.setQueryString(collationQueryStr); } try { queryComponent.prepare(checkResponse); if (docCollectionLimit > 0) { int f = checkResponse.getFieldFlags(); checkResponse.setFieldFlags(f |= SolrIndexSearcher.TERMINATE_EARLY); } queryComponent.process(checkResponse); hits = (Integer) checkResponse.rsp.getToLog().get("hits"); } catch (EarlyTerminatingCollectorException etce) { assert (docCollectionLimit > 0); assert 0 < etce.getNumberScanned(); assert 0 < etce.getNumberCollected(); if (etce.getNumberScanned() == maxDocId) { hits = etce.getNumberCollected(); } else { hits = (int) (((float) (maxDocId * etce.getNumberCollected())) / (float) etce.getNumberScanned()); } } catch (Exception e) { LOG.warn( "Exception trying to re-query to check if a spell check possibility would return any hits." + e); } finally { checkResponse.req.close(); } } if (hits > 0 || !verifyCandidateWithQuery) { collNo++; AlfrescoSpellCheckCollation collation = new AlfrescoSpellCheckCollation(); collation.setCollationQuery(aftsQuery); collation.setCollationQueryString(collationQueryStr); collation.setHits(hits); collation.setInternalRank( suggestionsMayOverlap ? ((possibility.rank * 1000) + possibility.index) : possibility.rank); NamedList<String> misspellingsAndCorrections = new NamedList<>(); for (SpellCheckCorrection corr : possibility.corrections) { misspellingsAndCorrections.add(corr.getOriginal().toString(), corr.getCorrection()); } collation.setMisspellingsAndCorrections(misspellingsAndCorrections); collations.add(collation); } if (LOG.isDebugEnabled()) { LOG.debug("Collation: " + aftsQuery + (verifyCandidateWithQuery ? (" will return " + hits + " hits.") : "")); } } return collations; }