List of usage examples for org.apache.solr.search DocSlice DocSlice
public DocSlice(int offset, int len, int[] docs, float[] scores, long matches, float maxScore)
From source file:lux.solr.XQueryComponent.java
License:Mozilla Public License
protected void evaluateQuery(ResponseBuilder rb, int start, int len) { String query = rb.getQueryString(); SolrQueryRequest req = rb.req;/* w ww . j a v a 2 s. c o m*/ SolrQueryResponse rsp = rb.rsp; if (StringUtils.isBlank(query)) { rsp.add("xpath-error", "query was blank"); return; } SolrParams params = req.getParams(); long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1); XQueryExecutable expr; LuxSearcher searcher = new LuxSearcher(rb.req.getSearcher()); DocWriter docWriter = new SolrDocWriter(this, rb.req.getCore()); Compiler compiler = solrIndexConfig.getCompiler(); SolrQueryContext context = new SolrQueryContext(this, req); if (rb.shards != null && rb.req.getParams().getBool("distrib", true)) { // This is a distributed request; pass in the ResponseBuilder so it will be // available to a subquery. context.setResponseBuilder(rb); // also capture the current set of shards shards = rb.shards; slices = rb.slices; } SolrSearchService searchService = new SolrSearchService(context, new LuxSearchQueryParser()); Evaluator eval = new Evaluator(compiler, searcher, docWriter, searchService); // track which evaluator we are using in a threadlocal container evalHolder.set(eval); TransformErrorListener errorListener = eval.getErrorListener(); try { URI baseURI = queryPath == null ? null : java.net.URI.create(queryPath); expr = compiler.compile(query, errorListener, baseURI, null); } catch (LuxException ex) { // ex.printStackTrace(); String err = formatError(query, errorListener); if (StringUtils.isEmpty(err)) { err = ex.getMessage(); } rsp.add("xpath-error", err); // don't close: this forces a commit() // evaluator.close(); return; } // SolrIndexSearcher.QueryResult result = new // SolrIndexSearcher.QueryResult(); NamedList<Object> xpathResults = new NamedList<Object>(); long tstart = System.currentTimeMillis(); int count = 0; bindRequestVariables(rb, req, expr, compiler, eval, context); Iterator<XdmItem> queryResults = eval.iterator(expr, context); String err = null; while (queryResults.hasNext()) { XdmItem xpathResult = queryResults.next(); if (++count < start) { continue; } if (count == 1 && !xpathResult.isAtomicValue()) { net.sf.saxon.s9api.QName name = ((XdmNode) xpathResult).getNodeName(); if (name != null && name.getNamespaceURI().equals(EXPATH_HTTP_NS) && name.getLocalName().equals("response")) { err = handleEXPathResponse(req, rsp, xpathResults, xpathResult); if (queryResults.hasNext()) { logger.warn( "Ignoring results following http:response, which should be the sole item in its result"); } break; } } err = safeAddResult(xpathResults, xpathResult); if (err != null) { xpathResult = null; break; } if ((len > 0 && xpathResults.size() >= len) || (timeAllowed > 0 && (System.currentTimeMillis() - tstart) > timeAllowed)) { break; } } ArrayList<TransformerException> errors = eval.getErrorListener().getErrors(); if (!errors.isEmpty()) { err = formatError(query, errors, eval.getQueryStats()); if (xpathResults.size() == 0) { xpathResults = null; // throw a 400 error; don't return partial // results } } if (err != null) { rsp.add("xpath-error", err); } if (rb.getResults() == null) { // create a dummy doc list if previous query processing didn't retrieve any docs // In distributed operation, there will be doc results, otherwise none. SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult(); result.setDocList(new DocSlice(0, 0, null, null, eval.getQueryStats().docCount, 0)); rb.setResult(result); rsp.add("response", rb.getResults().docList); } if (xpathResults != null) { rsp.add("xpath-results", xpathResults); if (logger.isDebugEnabled()) { logger.debug("retrieved: " + eval.getDocReader().getCacheMisses() + " docs, " + xpathResults.size() + " results, " + (System.currentTimeMillis() - tstart) + "ms"); } } else { logger.warn("xquery evaluation error: " + eval.getDocReader().getCacheMisses() + " docs, " + "0 results, " + (System.currentTimeMillis() - tstart) + "ms"); } if (err == null && context.isCommitPending()) { doCommit(); } }
From source file:net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.java
License:Open Source License
/** * conversion from a SolrQueryResponse (which is a solr-internal data format) to SolrDocumentList (which is a solrj-format) * The conversion is done inside the solrj api using the BinaryResponseWriter and a very complex unfolding process * via org.apache.solr.common.util.JavaBinCodec.marshal. * @param request/*from www .j a v a 2 s . c o m*/ * @param sqr * @return */ public SolrDocumentList SolrQueryResponse2SolrDocumentList(final SolrQueryRequest req, final SolrQueryResponse rsp) { SolrDocumentList sdl = new SolrDocumentList(); NamedList<?> nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; sdl.setNumFound(response == null ? 0 : response.matches()); sdl.setStart(response == null ? 0 : response.offset()); String originalName = Thread.currentThread().getName(); if (response != null) { try { SolrIndexSearcher searcher = req.getSearcher(); final int responseCount = response.size(); DocIterator iterator = response.iterator(); for (int i = 0; i < responseCount; i++) { int docid = iterator.nextDoc(); Thread.currentThread() .setName("EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList: " + docid); Document responsedoc = searcher.doc(docid, (Set<String>) null); SolrDocument sordoc = doc2SolrDoc(responsedoc); sdl.add(sordoc); } } catch (IOException e) { ConcurrentLog.logException(e); } } Thread.currentThread().setName(originalName); return sdl; }
From source file:opennlp.tools.similarity.apps.solr.IterativeSearchRequestHandler.java
License:Apache License
public DocList filterResultsBySyntMatchReduceDocSet(DocList docList, SolrQueryRequest req, SolrParams params) { //if (!docList.hasScores()) // return docList; int len = docList.size(); if (len < 1) // do nothing return docList; ParserChunker2MatcherProcessor pos = ParserChunker2MatcherProcessor.getInstance(); DocIterator iter = docList.iterator(); float[] syntMatchScoreArr = new float[len]; String requestExpression = req.getParamString(); String[] exprParts = requestExpression.split("&"); for (String part : exprParts) { if (part.startsWith("q=")) requestExpression = part;/* w w w. j av a 2 s. c o m*/ } String fieldNameQuery = StringUtils.substringBetween(requestExpression, "=", ":"); // extract phrase query (in double-quotes) String[] queryParts = requestExpression.split("\""); if (queryParts.length >= 2 && queryParts[1].length() > 5) requestExpression = queryParts[1].replace('+', ' '); else if (requestExpression.indexOf(":") > -1) {// still field-based expression requestExpression = requestExpression.replaceAll(fieldNameQuery + ":", "").replace('+', ' ') .replaceAll(" ", " ").replace("q=", ""); } if (fieldNameQuery == null) return docList; if (requestExpression == null || requestExpression.length() < 5 || requestExpression.split(" ").length < 3) return docList; int[] docIDsHits = new int[len]; IndexReader indexReader = req.getSearcher().getIndexReader(); List<Integer> bestMatchesDocIds = new ArrayList<Integer>(); List<Float> bestMatchesScore = new ArrayList<Float>(); List<Pair<Integer, Float>> docIdsScores = new ArrayList<Pair<Integer, Float>>(); try { for (int i = 0; i < docList.size(); ++i) { int docId = iter.nextDoc(); docIDsHits[i] = docId; Document doc = indexReader.document(docId); // get text for event String answerText = doc.get(fieldNameQuery); if (answerText == null) continue; SentencePairMatchResult matchResult = pos.assessRelevance(requestExpression, answerText); float syntMatchScore = new Double( parseTreeChunkListScorer.getParseTreeChunkListScore(matchResult.getMatchResult())) .floatValue(); bestMatchesDocIds.add(docId); bestMatchesScore.add(syntMatchScore); syntMatchScoreArr[i] = (float) syntMatchScore; //*iter.score(); System.out.println(" Matched query = '" + requestExpression + "' with answer = '" + answerText + "' | doc_id = '" + docId); System.out.println(" Match result = '" + matchResult.getMatchResult() + "' with score = '" + syntMatchScore + "';"); docIdsScores.add(new Pair(docId, syntMatchScore)); } } catch (CorruptIndexException e1) { // TODO Auto-generated catch block e1.printStackTrace(); //log.severe("Corrupt index"+e1); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); //log.severe("File read IO / index"+e1); } Collections.sort(docIdsScores, new PairComparable()); for (int i = 0; i < docIdsScores.size(); i++) { bestMatchesDocIds.set(i, docIdsScores.get(i).getFirst()); bestMatchesScore.set(i, docIdsScores.get(i).getSecond()); } System.out.println(bestMatchesScore); float maxScore = docList.maxScore(); // do not change int limit = docIdsScores.size(); int start = 0; DocSlice ds = null; ds = new DocSlice(start, limit, ArrayUtils.toPrimitive(bestMatchesDocIds.toArray(new Integer[0])), ArrayUtils.toPrimitive(bestMatchesScore.toArray(new Float[0])), bestMatchesDocIds.size(), maxScore); return ds; }
From source file:org.opencms.search.solr.CmsSolrIndex.java
License:Open Source License
/** * Performs the actual search.<p>/*from www.j av a 2 s . c o m*/ * * @param cms the current OpenCms context * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows * @param query the OpenCms Solr query * @param response the servlet response to write the query result to, may also be <code>null</code> * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case * @param filter the resource filter to use * * @return the found documents * * @throws CmsSearchException if something goes wrong * * @see #search(CmsObject, CmsSolrQuery, boolean) */ @SuppressWarnings("unchecked") public CmsSolrResultList search(CmsObject cms, final CmsSolrQuery query, boolean ignoreMaxRows, ServletResponse response, boolean ignoreSearchExclude, CmsResourceFilter filter) throws CmsSearchException { // check if the user is allowed to access this index checkOfflineAccess(cms); if (!ignoreSearchExclude) { query.addFilterQuery(CmsSearchField.FIELD_SEARCH_EXCLUDE + ":\"false\""); } int previousPriority = Thread.currentThread().getPriority(); long startTime = System.currentTimeMillis(); // remember the initial query SolrQuery initQuery = query.clone(); query.setHighlight(false); LocalSolrQueryRequest solrQueryRequest = null; try { // initialize the search context CmsObject searchCms = OpenCms.initCmsObject(cms); // change thread priority in order to reduce search impact on overall system performance if (getPriority() > 0) { Thread.currentThread().setPriority(getPriority()); } // the lists storing the found documents that will be returned List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(); SolrDocumentList solrDocumentList = new SolrDocumentList(); // Initialize rows, offset, end and the current page. int rows = query.getRows() != null ? query.getRows().intValue() : CmsSolrQuery.DEFAULT_ROWS.intValue(); if (!ignoreMaxRows && (rows > ROWS_MAX)) { rows = ROWS_MAX; } int start = query.getStart() != null ? query.getStart().intValue() : 0; int end = start + rows; int page = 0; if (rows > 0) { page = Math.round(start / rows) + 1; } // set the start to '0' and expand the rows before performing the query query.setStart(new Integer(0)); query.setRows(new Integer((5 * rows * page) + start)); // perform the Solr query and remember the original Solr response QueryResponse queryResponse = m_solr.query(query); long solrTime = System.currentTimeMillis() - startTime; // initialize the counts long hitCount = queryResponse.getResults().getNumFound(); start = -1; end = -1; if ((rows > 0) && (page > 0) && (hitCount > 0)) { // calculate the final size of the search result start = rows * (page - 1); end = start + rows; // ensure that both i and n are inside the range of foundDocuments.size() start = new Long((start > hitCount) ? hitCount : start).intValue(); end = new Long((end > hitCount) ? hitCount : end).intValue(); } else { // return all found documents in the search result start = 0; end = new Long(hitCount).intValue(); } long visibleHitCount = hitCount; float maxScore = 0; // If we're using a postprocessor, (re-)initialize it before using it if (m_postProcessor != null) { m_postProcessor.init(); } // process found documents List<CmsSearchResource> allDocs = new ArrayList<CmsSearchResource>(); int cnt = 0; for (int i = 0; (i < queryResponse.getResults().size()) && (cnt < end); i++) { try { SolrDocument doc = queryResponse.getResults().get(i); CmsSolrDocument searchDoc = new CmsSolrDocument(doc); if (needsPermissionCheck(searchDoc)) { // only if the document is an OpenCms internal resource perform the permission check CmsResource resource = filter == null ? getResource(searchCms, searchDoc) : getResource(searchCms, searchDoc, filter); if (resource != null) { // permission check performed successfully: the user has read permissions! if (cnt >= start) { if (m_postProcessor != null) { doc = m_postProcessor.process(searchCms, resource, (SolrInputDocument) searchDoc.getDocument()); } resourceDocumentList.add(new CmsSearchResource(resource, searchDoc)); if (null != doc) { solrDocumentList.add(doc); } maxScore = maxScore < searchDoc.getScore() ? searchDoc.getScore() : maxScore; } allDocs.add(new CmsSearchResource(resource, searchDoc)); cnt++; } else { visibleHitCount--; } } else { // if permission check is not required for this index, // add a pseudo resource together with document to the results resourceDocumentList.add(new CmsSearchResource(PSEUDO_RES, searchDoc)); solrDocumentList.add(doc); maxScore = maxScore < searchDoc.getScore() ? searchDoc.getScore() : maxScore; cnt++; } } catch (Exception e) { // should not happen, but if it does we want to go on with the next result nevertheless LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); } } // the last documents were all secret so let's take the last found docs if (resourceDocumentList.isEmpty() && (allDocs.size() > 0)) { page = Math.round(allDocs.size() / rows) + 1; int showCount = allDocs.size() % rows; showCount = showCount == 0 ? rows : showCount; start = allDocs.size() - new Long(showCount).intValue(); end = allDocs.size(); if (allDocs.size() > start) { resourceDocumentList = allDocs.subList(start, end); for (CmsSearchResource r : resourceDocumentList) { maxScore = maxScore < r.getDocument().getScore() ? r.getDocument().getScore() : maxScore; solrDocumentList.add(((CmsSolrDocument) r.getDocument()).getSolrDocument()); } } } long processTime = System.currentTimeMillis() - startTime - solrTime; // create and return the result solrDocumentList.setStart(start); solrDocumentList.setMaxScore(new Float(maxScore)); solrDocumentList.setNumFound(visibleHitCount); queryResponse.getResponse().setVal(queryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0), solrDocumentList); queryResponse.getResponseHeader().setVal(queryResponse.getResponseHeader().indexOf(QUERY_TIME_NAME, 0), new Integer(new Long(System.currentTimeMillis() - startTime).intValue())); long highlightEndTime = System.currentTimeMillis(); SolrCore core = m_solr instanceof EmbeddedSolrServer ? ((EmbeddedSolrServer) m_solr).getCoreContainer().getCore(getCoreName()) : null; CmsSolrResultList result = null; try { SearchComponent highlightComponenet = null; if (core != null) { highlightComponenet = core.getSearchComponent("highlight"); solrQueryRequest = new LocalSolrQueryRequest(core, queryResponse.getResponseHeader()); } SolrQueryResponse solrQueryResponse = null; if (solrQueryRequest != null) { // create and initialize the solr response solrQueryResponse = new SolrQueryResponse(); solrQueryResponse.setAllValues(queryResponse.getResponse()); int paramsIndex = queryResponse.getResponseHeader().indexOf(HEADER_PARAMS_NAME, 0); NamedList<Object> header = null; Object o = queryResponse.getResponseHeader().getVal(paramsIndex); if (o instanceof NamedList) { header = (NamedList<Object>) o; header.setVal(header.indexOf(CommonParams.ROWS, 0), new Integer(rows)); header.setVal(header.indexOf(CommonParams.START, 0), new Long(start)); } // set the OpenCms Solr query as parameters to the request solrQueryRequest.setParams(initQuery); // perform the highlighting if ((header != null) && (initQuery.getHighlight()) && (highlightComponenet != null)) { header.add(HighlightParams.HIGHLIGHT, "on"); if ((initQuery.getHighlightFields() != null) && (initQuery.getHighlightFields().length > 0)) { header.add(HighlightParams.FIELDS, CmsStringUtil.arrayAsString(initQuery.getHighlightFields(), ",")); } String formatter = initQuery.getParams(HighlightParams.FORMATTER) != null ? initQuery.getParams(HighlightParams.FORMATTER)[0] : null; if (formatter != null) { header.add(HighlightParams.FORMATTER, formatter); } if (initQuery.getHighlightFragsize() != 100) { header.add(HighlightParams.FRAGSIZE, new Integer(initQuery.getHighlightFragsize())); } if (initQuery.getHighlightRequireFieldMatch()) { header.add(HighlightParams.FIELD_MATCH, new Boolean(initQuery.getHighlightRequireFieldMatch())); } if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(initQuery.getHighlightSimplePost())) { header.add(HighlightParams.SIMPLE_POST, initQuery.getHighlightSimplePost()); } if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(initQuery.getHighlightSimplePre())) { header.add(HighlightParams.SIMPLE_PRE, initQuery.getHighlightSimplePre()); } if (initQuery.getHighlightSnippets() != 1) { header.add(HighlightParams.SNIPPETS, new Integer(initQuery.getHighlightSnippets())); } ResponseBuilder rb = new ResponseBuilder(solrQueryRequest, solrQueryResponse, Collections.singletonList(highlightComponenet)); try { rb.doHighlights = true; DocListAndSet res = new DocListAndSet(); SchemaField idField = OpenCms.getSearchManager().getSolrServerConfiguration() .getSolrSchema().getUniqueKeyField(); int[] luceneIds = new int[rows]; int docs = 0; for (SolrDocument doc : solrDocumentList) { String idString = (String) doc.getFirstValue(CmsSearchField.FIELD_ID); int id = solrQueryRequest.getSearcher().getFirstMatch( new Term(idField.getName(), idField.getType().toInternal(idString))); luceneIds[docs++] = id; } res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0); rb.setResults(res); rb.setQuery(QParser.getParser(initQuery.getQuery(), null, solrQueryRequest).getQuery()); rb.setQueryString(initQuery.getQuery()); highlightComponenet.prepare(rb); highlightComponenet.process(rb); highlightComponenet.finishStage(rb); } catch (Exception e) { LOG.error(e.getMessage() + " in query: " + initQuery, new Exception(e)); } // Make highlighting also available via the CmsSolrResultList queryResponse.setResponse(solrQueryResponse.getValues()); highlightEndTime = System.currentTimeMillis(); } } result = new CmsSolrResultList(initQuery, queryResponse, solrDocumentList, resourceDocumentList, start, new Integer(rows), end, page, visibleHitCount, new Float(maxScore), startTime, highlightEndTime); if (LOG.isDebugEnabled()) { Object[] logParams = new Object[] { new Long(System.currentTimeMillis() - startTime), new Long(result.getNumFound()), new Long(solrTime), new Long(processTime), new Long(result.getHighlightEndTime() != 0 ? result.getHighlightEndTime() - startTime : 0) }; LOG.debug(query.toString() + "\n" + Messages.get().getBundle().key(Messages.LOG_SOLR_SEARCH_EXECUTED_5, logParams)); } if (response != null) { writeResp(response, solrQueryRequest, solrQueryResponse); } } finally { if (solrQueryRequest != null) { solrQueryRequest.close(); } if (core != null) { core.close(); } } return result; } catch (Exception e) { throw new CmsSearchException(Messages.get().container(Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, CmsEncoder.decode(query.toString()), e), e); } finally { if (solrQueryRequest != null) { solrQueryRequest.close(); } // re-set thread to previous priority Thread.currentThread().setPriority(previousPriority); } }
From source file:org.opensextant.solrtexttagger.TaggerRequestHandler.java
License:Open Source License
private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException { //Now we must supply a Solr DocList and add it to the response. // Typically this is gotten via a SolrIndexSearcher.search(), but in this case we // know exactly what documents to return, the order doesn't matter nor does // scoring./*from w w w . ja va 2 s . c om*/ // Ideally an implementation of DocList could be directly implemented off // of a BitSet, but there are way too many methods to implement for a minor // payoff. int matchDocs = matchDocIdsBS.cardinality(); int[] docIds = new int[Math.min(rows, matchDocs)]; DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1); for (int i = 0; i < docIds.length; i++) { docIds[i] = docIdIter.nextDoc(); } return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f); }