List of usage examples for org.apache.solr.common.params CursorMarkParams CURSOR_MARK_START
String CURSOR_MARK_START
To view the source code for org.apache.solr.common.params CursorMarkParams CURSOR_MARK_START.
Click Source Link
From source file:bamboo.trove.rule.RuleChangeUpdateManager.java
License:Apache License
private void processQuery(SolrQuery query, WorkLog workLog) throws SolrServerException, IOException { log.debug("Query for rule : {}", query.toString()); Timer.Context context = getTimer(getName() + ".processQuery").time(); // need to commit here so that we can ignore documents just processed client.commit();/*w ww. j a va2s . c o m*/ boolean more = true; String cursor = CursorMarkParams.CURSOR_MARK_START; while (more) { query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursor); Timer.Context contextQuery = getTimer(getName() + ".query").time(); QueryResponse response = client.query(query); workLog.ranSearch(); SolrDocumentList results = response.getResults(); log.debug("Found {} (of {} docs) in QT = {} ms", results.size(), results.getNumFound(), response.getQTime()); String nextCursor = response.getNextCursorMark(); if (nextCursor == null || cursor.equals(nextCursor)) { more = false; } distributeResponse(results, workLog); cursor = nextCursor; contextQuery.stop(); } // We do this at a higher level too, so this would seem redundant. There is a trade-off. Allowing parallelism // between rules means rules can sometimes be re-processed redundantly. The higher level waitUntilCaughtUp() will // ensure we never process rules at the same time rules are being changed. // By doing a wait here as well however, we can collect accurate statistics about how much actual write activity we // are really generating by passing the workLog into the work pool. // When we have a better awareness of the typical work patterns it might be worth disabling this method call and // then stop collecting the metrics to improve throughput. waitUntilCaughtUp(); context.stop(); }
From source file:com.frank.search.solr.core.query.result.DelegatingCursor.java
License:Apache License
protected DelegatingCursor(SolrQuery query) { this(query, CursorMarkParams.CURSOR_MARK_START); }
From source file:com.frank.search.solr.core.query.result.DelegatingCursor.java
License:Apache License
protected DelegatingCursor(SolrQuery query, String initalCursorMark) { this.referenceQuery = query; this.cursorMark = StringUtils.hasText(initalCursorMark) ? initalCursorMark : CursorMarkParams.CURSOR_MARK_START; this.state = State.REDAY; this.delegate = Collections.<T>emptyList().iterator(); }
From source file:com.hurence.logisland.service.solr.api.SolrClientService.java
License:Apache License
@Override public void copyCollection(String reindexScrollTimeout, String src, String dst) throws DatastoreClientServiceException { SolrQuery solrQuery = new SolrQuery(); solrQuery.setRows(1000);//from w w w . j a va2s. co m solrQuery.setQuery("*:*"); solrQuery.addSort("id", SolrQuery.ORDER.asc); // Pay attention to this line String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; QueryResponse response; try { do { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); response = getClient().query(src, solrQuery); List<SolrInputDocument> documents = new ArrayList<>(); for (SolrDocument document : response.getResults()) { SolrInputDocument inputDocument = getConverter().toSolrInputDocument(document); inputDocument.removeField("_version_"); documents.add(inputDocument); } getClient().add(dst, documents); } while (cursorMark.equals(response.getNextCursorMark())); getClient().commit(dst); } catch (Exception e) { throw new DatastoreClientServiceException(e); } }
From source file:it.damore.solr.importexport.App.java
License:Open Source License
/** * @param client/*from w ww . j a v a 2 s . co m*/ * @param outputFile * @throws SolrServerException * @throws IOException */ private static void readAllDocuments(HttpSolrClient client, File outputFile) throws SolrServerException, IOException { SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery("*:*"); if (config.getFilterQuery() != null) { solrQuery.addFilterQuery(config.getFilterQuery()); } solrQuery.setRows(0); solrQuery.addSort(config.getUniqueKey(), ORDER.asc); // Pay attention to this line String cursorMark = CursorMarkParams.CURSOR_MARK_START; TimeZone.setDefault(TimeZone.getTimeZone("UTC")); // objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); DateFormat df = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:sss'Z'"); objectMapper.setDateFormat(df); objectMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); QueryResponse r = client.query(solrQuery); long nDocuments = r.getResults().getNumFound(); logger.info("Found " + nDocuments + " documents"); if (!config.getDryRun()) { logger.info("Creating " + config.getFileName()); Set<SkipField> skipFieldsEquals = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.EQUAL).collect(Collectors.toSet()); Set<SkipField> skipFieldsStartWith = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.STARTS_WITH).collect(Collectors.toSet()); Set<SkipField> skipFieldsEndWith = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.ENDS_WITH).collect(Collectors.toSet()); try (PrintWriter pw = new PrintWriter(outputFile)) { solrQuery.setRows(200); boolean done = false; while (!done) { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = client.query(solrQuery); String nextCursorMark = rsp.getNextCursorMark(); for (SolrDocument d : rsp.getResults()) { skipFieldsEquals.forEach(f -> d.removeFields(f.getText())); if (skipFieldsStartWith.size() > 0 || skipFieldsEndWith.size() > 0) { Map<String, Object> collect = d.entrySet().stream() .filter(e -> !skipFieldsStartWith.stream() .filter(f -> e.getKey().startsWith(f.getText())).findFirst() .isPresent()) .filter(e -> !skipFieldsEndWith.stream() .filter(f -> e.getKey().endsWith(f.getText())).findFirst().isPresent()) .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); pw.write(objectMapper.writeValueAsString(collect)); } else { pw.write(objectMapper.writeValueAsString(d)); } pw.write("\n"); } if (cursorMark.equals(nextCursorMark)) { done = true; } cursorMark = nextCursorMark; } } } }
From source file:org.apache.metron.solr.dao.SolrMetaAlertSearchDao.java
License:Apache License
@Override public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { if (guid == null || guid.trim().isEmpty()) { throw new InvalidSearchException("Guid cannot be empty"); }//from w w w . ja v a 2s. c o m // Searches for all alerts containing the meta alert guid in it's "metalerts" array // The query has to match the parentFilter to avoid errors. Guid must also be explicitly // included. String activeClause = MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString(); String guidClause = Constants.GUID + ":" + guid; String fullClause = "{!parent which=" + activeClause + "}" + guidClause; String metaalertTypeClause = config.getSourceTypeField() + ":" + MetaAlertConstants.METAALERT_TYPE; SolrQuery solrQuery = new SolrQuery().setQuery(fullClause) .setFields("*", "[child parentFilter=" + metaalertTypeClause + " limit=999]") .addSort(Constants.GUID, SolrQuery.ORDER.asc); // Just do basic sorting to track where we are // Use Solr's Cursors to handle the paging, rather than doing it manually. List<SearchResult> allResults = new ArrayList<>(); try { String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; while (!done) { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = solrClient.query(METAALERTS_COLLECTION, solrQuery); String nextCursorMark = rsp.getNextCursorMark(); rsp.getResults().stream() .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null, solrSearchDao.getAccessConfig().getIndexSupplier())) .forEachOrdered(allResults::add); if (cursorMark.equals(nextCursorMark)) { done = true; } cursorMark = nextCursorMark; } } catch (IOException | SolrServerException e) { throw new InvalidSearchException("Unable to complete search", e); } SearchResponse searchResponse = new SearchResponse(); searchResponse.setResults(allResults); searchResponse.setTotal(allResults.size()); return searchResponse; }
From source file:org.geotools.data.solr.SolrFeatureReader.java
License:Open Source License
/** * Creates the feature reader for SOLR store <br> * The feature reader use SOLR CURSOR to paginate request, so multiple SOLR query will be * executed/*from w w w . j av a2s .c o m*/ * * @param featureType the feature type to query * @param solrUrl the URL of SOLR server * @param solrQuery the SOLR query to execute * @param solrDataStore the SOLR store * @throws SolrServerException */ public SolrFeatureReader(SimpleFeatureType featureType, HttpSolrServer server, SolrQuery solrQuery, SolrDataStore solrDataStore) throws SolrServerException { this.featureType = featureType; this.solrQuery = solrQuery; this.wktReader = new WKTReader(); this.solrDataStore = solrDataStore; this.pkey = solrDataStore.getPrimaryKey(featureType.getTypeName()); this.builder = new SimpleFeatureBuilder(featureType); this.server = server; // Add always pk as field if not already present if (solrQuery.getFields() != null && !solrQuery.getFields().contains(pkey.getName())) { solrQuery.addField(pkey.getName()); } // Can't use pagination with start, then first query for cursor mark of start if (solrQuery.getStart() != null && solrQuery.getStart() > 0) { cursorMark = getCursorMarkForStart(server, solrQuery); } else { cursorMark = CursorMarkParams.CURSOR_MARK_START; } solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = server.query(solrQuery); if (this.solrDataStore.getLogger().isLoggable(Level.FINE)) { this.solrDataStore.getLogger().log(Level.FINE, "SOLR query done: " + solrQuery.toString()); } this.solrDocIterator = rsp.getResults().iterator(); nextCursorMark = rsp.getNextCursorMark(); counter = 0; }
From source file:org.geotools.data.solr.SolrFeatureReader.java
License:Open Source License
private String getCursorMarkForStart(HttpSolrServer server, SolrQuery solrQuery) throws SolrServerException { Integer prevRows = solrQuery.getRows(); solrQuery.setRows(solrQuery.getStart()); solrQuery.setStart(0);/*from w ww . j a va 2s .co m*/ solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, CursorMarkParams.CURSOR_MARK_START); QueryResponse rsp = server.query(solrQuery); if (this.solrDataStore.getLogger().isLoggable(Level.FINE)) { this.solrDataStore.getLogger().log(Level.FINE, "SOLR query done: " + solrQuery.toString()); } String nextC = rsp.getNextCursorMark(); solrQuery.setRows(prevRows); return nextC; }
From source file:org.pentaho.di.trans.steps.solrinput.SolrInput.java
License:Apache License
/** * Once the transformation starts executing, the processRow() method is called repeatedly * by PDI for as long as it returns true. To indicate that a step has finished processing rows * this method must call setOutputDone() and return false; * //from w w w. jav a 2 s . c om * Steps which process incoming rows typically call getRow() to read a single row from the * input stream, change or add row content, call putRow() to pass the changed row on * and return true. If getRow() returns null, no more rows are expected to come in, * and the processRow() implementation calls setOutputDone() and returns false to * indicate that it is done too. * * Steps which generate rows typically construct a new row Object[] using a call to * RowDataUtil.allocateRowData(numberOfFields), add row content, and call putRow() to * pass the new row on. Above process may happen in a loop to generate multiple rows, * at the end of which processRow() would call setOutputDone() and return false; * * @param smi the step meta interface containing the step settings * @param sdi the step data interface that should be used to store * * @return true to indicate that the function should be called again, false if the step is done */ public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { if (first) { first = false; // Create the output row meta-data data.outputRowMeta = new RowMeta(); meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore); // For String to <type> conversions, we allocate a conversion meta data row as well... data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING); // get real values boolean tryCursor = true; Integer startRecord = 0; Integer chunkRowSize = 20; String realURL = meta.getURL(); String realQ = meta.getQ(); String realSort = meta.getSort(); String realCursor = meta.getCursor(); String realFq = meta.getFq(); String realFl = meta.getFl(); String realFacetQuery = meta.getFacetQuery(); String realFacetField = meta.getFacetField(); /* Send and Get the report */ SolrQuery query = new SolrQuery(); query.set("rows", chunkRowSize); if (realQ != null && !realQ.equals("")) { query.set("q", realQ); } if (realSort != null && !realSort.equals("")) { query.set("sort", realSort); } else { tryCursor = false; } if (realCursor != null && !realCursor.equals("")) { if (realCursor.equals("No")) { tryCursor = false; } } if (realFl != null && !realFl.equals("")) { query.set("fl", realFl); } if (realFq != null && !realFq.equals("")) { query.set("fq", realFq); } if (realFacetField != null && !realFacetField.equals("")) { //TODO incorporate multiple facet fields at once //String[] facetFields = realFacetField.split("\\s*,\\s*"); //for(int i =0; i < facetFields.length; i++){ query.addFacetField(realFacetField); //} query.setFacet(true); query.setFacetLimit(-1); query.setFacetMinCount(0); query.setFacetSort("count"); query.set("rows", 0); tryCursor = false; data.facetRequested = true; } if (realFacetQuery != null && !realFacetQuery.equals("")) { query.addFacetQuery(realFacetQuery); } // You can't use "TimeAllowed" with "CursorMark" // The documentation says "Values <= 0 mean // no time restriction", so setting to 0. query.setTimeAllowed(0); HttpSolrServer solr = new HttpSolrServer(realURL); String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; QueryResponse rsp = null; while (!done) { if (tryCursor) { query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); } else { query.setStart(startRecord); } try { rsp = solr.query(query); } catch (SolrServerException e) { e.printStackTrace(); } if (data.facetRequested) { data.facetCountName = rsp.getFacetFields().get(0).getName(); data.facetCounts = rsp.getFacetFields().get(0).getValues(); done = true; } else { SolrDocumentList theseDocs = rsp.getResults(); for (SolrDocument doc : theseDocs) { data.documentList.add(doc); } } if (tryCursor) { String nextCursorMark = rsp.getNextCursorMark(); if (cursorMark.equals(nextCursorMark)) { done = true; } else { cursorMark = nextCursorMark; } } else { startRecord = startRecord + chunkRowSize; if (startRecord >= rsp.getResults().getNumFound()) { done = true; } } } } Object[] outputRowData = null; try { if (data.facetRequested) { // get one row if we can if (data.facetCounts.size() - 1 < data.recordIndex) { setOutputDone(); return false; } FacetField.Count facetRecord = data.facetCounts.get(data.recordIndex); outputRowData = prepareFacetRecord(facetRecord); } else { // get one row if we can if (data.documentList.size() - 1 < data.recordIndex) { setOutputDone(); return false; } SolrDocument record = data.documentList.get(data.recordIndex); outputRowData = prepareRecord(record); } putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s); data.recordIndex++; return true; } catch (KettleException e) { boolean sendToErrorRow = false; String errorMessage = null; if (getStepMeta().isDoingErrorHandling()) { sendToErrorRow = true; errorMessage = e.toString(); } else { logError(BaseMessages.getString(PKG, "SolrInput.log.Exception", e.getMessage())); logError(Const.getStackTracker(e)); setErrors(1); stopAll(); setOutputDone(); // signal end to receiver(s) return false; } if (sendToErrorRow) { // Simply add this row to the error row putError(getInputRowMeta(), outputRowData, 1, errorMessage, null, "SolrInput001"); } } return true; }
From source file:org.pentaho.di.ui.trans.steps.solrinput.SolrInputDialog.java
License:Apache License
private void getFields() { try {//w ww . ja v a 2 s . com SolrInputMeta meta = new SolrInputMeta(); getInfo(meta); // clear the current fields grid wFields.removeAll(); // get real values boolean tryCursor = true; boolean facetRequested = false; Integer startRecord = 0; Integer chunkRowSize = 20; String realURL = transMeta.environmentSubstitute(meta.getURL()); String realQ = transMeta.environmentSubstitute(meta.getQ()); String realSort = transMeta.environmentSubstitute(meta.getSort()); String realCursor = transMeta.environmentSubstitute(meta.getCursor()); String realFq = transMeta.environmentSubstitute(meta.getFq()); String realFl = transMeta.environmentSubstitute(meta.getFl()); String realFacetQuery = transMeta.environmentSubstitute(meta.getFacetQuery()); String realFacetField = transMeta.environmentSubstitute(meta.getFacetField()); /* Send and Get the report */ SolrQuery query = new SolrQuery(); query.set("rows", chunkRowSize); if (realQ != null && !realQ.equals("")) { query.set("q", realQ); } if (realSort != null && !realSort.equals("")) { query.set("sort", realSort); } else { tryCursor = false; } if (realCursor != null && !realCursor.equals("")) { if (realCursor.equals("No")) { tryCursor = false; } } if (realFl != null && !realFl.equals("")) { query.set("fl", realFl); } if (realFq != null && !realFq.equals("")) { query.set("fq", realFq); } if (realFacetField != null && !realFacetField.equals("")) { //TODO incorporate multiple facet fields at once //String[] facetFields = realFacetField.split("\\s*,\\s*"); //for(int i =0; i < facetFields.length; i++){ query.addFacetField(realFacetField); //} query.setFacet(true); query.setFacetLimit(-1); query.setFacetMinCount(0); query.setFacetSort("count"); query.set("rows", 0); tryCursor = false; facetRequested = true; } if (realFacetQuery != null && !realFacetQuery.equals("")) { query.addFacetQuery(realFacetQuery); } // You can't use "TimeAllowed" with "CursorMark" // The documentation says "Values <= 0 mean // no time restriction", so setting to 0. query.setTimeAllowed(0); HttpSolrServer solr = new HttpSolrServer(realURL); String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; List<String> headerNames = new ArrayList<String>(); QueryResponse rsp = null; while (!done) { if (tryCursor) { query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); } else { query.setStart(startRecord); } try { rsp = solr.query(query); } catch (SolrServerException e) { e.printStackTrace(); } if (facetRequested) { headerNames.add(rsp.getFacetFields().get(0).getName()); headerNames.add("count"); done = true; } else { SolrDocumentList docs = rsp.getResults(); for (SolrDocument doc : docs) { Collection<String> thisNamesArray = doc.getFieldNames(); String[] a = thisNamesArray.toArray(new String[thisNamesArray.size()]); for (int j = 0; j < a.length; j++) { if (!headerNames.contains(a[j])) { headerNames.add(a[j]); } } } } if (tryCursor) { String nextCursorMark = rsp.getNextCursorMark(); if (cursorMark.equals(nextCursorMark)) { done = true; } else { cursorMark = nextCursorMark; } } else { startRecord = startRecord + chunkRowSize; if (startRecord >= rsp.getResults().getNumFound()) { done = true; } } } getTableView().table.setItemCount(headerNames.size()); for (int j = 0; j < headerNames.size(); j++) { TableItem item = getTableView().table.getItem(j); item.setText(1, headerNames.get(j)); } wFields.removeEmptyRows(); wFields.setRowNums(); wFields.optWidth(true); getInput().setChanged(); } catch (Exception e) { new ErrorDialog(shell, BaseMessages.getString(PKG, "SolrInputMeta.ErrorRetrieveData.DialogTitle"), BaseMessages.getString(PKG, "SolrInputMeta.ErrorRetrieveData.DialogMessage"), e); } }