List of usage examples for org.apache.solr.common SolrDocument removeFields
public boolean removeFields(String name)
From source file:com.github.fengtan.sophie.toolbars.DocumentsToolbar.java
License:Open Source License
/** * Populate toolbar with buttons./* w w w.j av a2 s . c o m*/ * * @param composite * Parent composite. */ private void initToolbar(final Composite composite) { Display display = composite.getDisplay(); ClassLoader loader = getClass().getClassLoader(); // Instantiate toolbar. ToolBar toolBar = new ToolBar(composite, SWT.BORDER); // Instantiate images. imgRefresh = new Image(display, loader.getResourceAsStream("toolbar/refresh.png")); imgAdd = new Image(display, loader.getResourceAsStream("toolbar/add.png")); imgDelete = new Image(display, loader.getResourceAsStream("toolbar/delete.png")); imgClone = new Image(display, loader.getResourceAsStream("toolbar/clone.png")); imgUpload = new Image(display, loader.getResourceAsStream("toolbar/upload.png")); imgAddField = new Image(display, loader.getResourceAsStream("toolbar/add_field.png")); imgClear = new Image(display, loader.getResourceAsStream("toolbar/clear.png")); imgCommit = new Image(display, loader.getResourceAsStream("toolbar/commit.png")); imgOptimize = new Image(display, loader.getResourceAsStream("toolbar/optimize.png")); imgExport = new Image(display, loader.getResourceAsStream("toolbar/export.png")); imgBackup = new Image(display, loader.getResourceAsStream("toolbar/backup.png")); imgRestore = new Image(display, loader.getResourceAsStream("toolbar/restore.png")); // Instantiate buttons. itemRefresh = new ToolItem(toolBar, SWT.PUSH); itemRefresh.setImage(imgRefresh); itemRefresh.setText("Refresh"); itemRefresh.setToolTipText("Refresh from Solr: this will wipe out local modifications"); itemRefresh.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { try { table.refresh(); } catch (SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to refresh documents from Solr server", e)); } } }); new ToolItem(toolBar, SWT.SEPARATOR); itemAdd = new ToolItem(toolBar, SWT.PUSH); itemAdd.setImage(imgAdd); itemAdd.setText("Add"); itemAdd.setToolTipText("Add new document"); itemAdd.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent e) { table.addDocument(new SolrDocument()); } }); itemDelete = new ToolItem(toolBar, SWT.PUSH); itemDelete.setImage(imgDelete); itemDelete.setText("Delete"); itemDelete.setToolTipText("Delete document"); itemDelete.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent e) { table.deleteSelectedDocument(); } }); itemDelete.setEnabled(false); itemClone = new ToolItem(toolBar, SWT.PUSH); itemClone.setImage(imgClone); itemClone.setText("Clone"); itemClone.setToolTipText("Clone document"); itemClone.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { SolrDocument document = table.getSelectedDocument(); if (document == null) { return; } // Unset the unique key field so we don't have two rows // describing the same Solr document. try { String uniqueKey = SolrUtils.getRemoteUniqueField(); document.removeFields(uniqueKey); } catch (SophieException e) { Sophie.log.warn("Unable to unset unique key on cloned document"); } // Add cloned document. table.addDocument(document); } }); itemClone.setEnabled(false); itemUpload = new ToolItem(toolBar, SWT.PUSH); itemUpload.setImage(imgUpload); itemUpload.setText("Upload"); itemUpload.setToolTipText("Upload local modifications to Solr"); itemUpload.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { try { table.upload(); } catch (SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to upload local modifications to Solr", e)); } } }); itemUpload.setEnabled(false); new ToolItem(toolBar, SWT.SEPARATOR); itemAddField = new ToolItem(toolBar, SWT.PUSH); itemAddField.setImage(imgAddField); itemAddField.setText("Add field"); itemAddField.setToolTipText("Add new field"); itemAddField.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { // Get field schema fields. Map<String, FieldInfo> fields; try { fields = SolrUtils.getRemoteSchemaFields(); } catch (SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to fetch schema fields", e)); return; } // Remove universal pattern which is not useful and will match // any field name. fields.remove("*"); // Remove fields already displayed in the table. Collection<String> excludedFieldNames = table.getFieldNames(); for (String existingFieldName : excludedFieldNames) { fields.remove(existingFieldName); } // Extract and sort field names. Set<String> fieldNames = fields.keySet(); String[] fieldNamesArray = new String[fieldNames.size()]; fieldNames.toArray(fieldNamesArray); Arrays.sort(fieldNamesArray); // Prompt user for new field name. FieldValidator validator = new FieldValidator(fields, excludedFieldNames); CComboDialog dialog = new CComboDialog(composite.getShell(), "Add new field", "Field name:", fieldNamesArray, validator); dialog.open(); if (dialog.getReturnCode() != IDialogConstants.OK_ID) { return; } // Add new field. String fieldName = dialog.getValue(); FieldInfo field = validator.getMatchingField(fieldName); table.addField(fieldName, field); } }); itemClear = new ToolItem(toolBar, SWT.PUSH); itemClear.setImage(imgClear); itemClear.setText("Clear"); itemClear.setToolTipText("Clear index"); itemClear.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { MessageBox messageBox = new MessageBox(composite.getShell(), SWT.ICON_QUESTION | SWT.YES | SWT.NO); messageBox.setText("Clear index"); messageBox.setMessage( "Do you really want to clear the index? This will remove all documents from the index."); int response = messageBox.open(); if (response == SWT.YES) { try { Sophie.client.deleteByQuery("*:*"); Sophie.client.commit(); table.refresh(); } catch (SolrServerException | IOException | SolrException | SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to clear index", e)); } } } }); itemCommit = new ToolItem(toolBar, SWT.PUSH); itemCommit.setImage(imgCommit); itemCommit.setText("Commit"); itemCommit.setToolTipText("Commit index"); itemCommit.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { try { Sophie.client.commit(); table.refresh(); } catch (SolrServerException | IOException | SolrException | SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to commit index", e)); } } }); itemOptimize = new ToolItem(toolBar, SWT.PUSH); itemOptimize.setImage(imgOptimize); itemOptimize.setText("Optimize"); itemOptimize.setToolTipText("Optimize index"); itemOptimize.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { MessageBox messageBox = new MessageBox(composite.getShell(), SWT.ICON_QUESTION | SWT.YES | SWT.NO); messageBox.setText("Optimize index"); messageBox.setMessage( "Do you really want to optimize the index? If the index is highly segmented, this may take several hours and will slow down requests."); int response = messageBox.open(); if (response == SWT.YES) { try { Sophie.client.optimize(); // Optimizing drops obsolete documents, obsolete facet // values, etc so we need to refresh the table. table.refresh(); } catch (SolrServerException | IOException | SolrException | SophieException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to optimize index", e)); } } } }); new ToolItem(toolBar, SWT.SEPARATOR); itemExport = new ToolItem(toolBar, SWT.PUSH); itemExport.setImage(imgExport); itemExport.setText("Export"); itemExport.setToolTipText("Export as CSV file"); itemExport.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { try { table.export(); } catch (SophieException e) { ExceptionDialog.open(composite.getShell(), e); } } }); itemBackup = new ToolItem(toolBar, SWT.PUSH); itemBackup.setImage(imgBackup); itemBackup.setText("Backup"); itemBackup.setToolTipText("Make a backup of the index"); itemBackup.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { InputDialog dialog = new InputDialog(composite.getShell(), "Make a backup of the index", "Backup name (leave empty to use a timestamp):", null, null); dialog.open(); if (dialog.getReturnCode() != IDialogConstants.OK_ID) { return; } String backupName = dialog.getValue(); ModifiableSolrParams params = new ModifiableSolrParams(); params.set("command", "backup"); params.set("name", backupName); QueryRequest request = new QueryRequest(params); request.setPath("/replication"); try { NamedList<Object> response = Sophie.client.request(request); // org.apache.solr.handler.ReplicationHandler.OK_STATUS is // "OK". if (StringUtils.equals(response.get("status").toString(), "OK")) { MessageBox messageBox = new MessageBox(composite.getShell(), SWT.ICON_INFORMATION | SWT.OK); messageBox.setText("Backup started"); messageBox.setMessage( "Backup started and will be saved into Solr data directory under the name \"snapshot." + (StringUtils.isEmpty(backupName) ? "<timestamp>" : backupName) + "\"."); messageBox.open(); } else { MessageBox messageBox = new MessageBox(composite.getShell(), SWT.ICON_ERROR | SWT.OK); messageBox.setText("Backup error"); messageBox.setMessage("Unable to backup the index."); messageBox.open(); } } catch (SolrServerException | IOException | SolrException e) { ExceptionDialog.open(composite.getShell(), new SophieException("Unable to create backup \"" + backupName + "\"", e)); } } }); itemRestore = new ToolItem(toolBar, SWT.PUSH); itemRestore.setImage(imgRestore); itemRestore.setText("Restore"); itemRestore.setToolTipText("Restore index from a backup"); itemRestore.addSelectionListener(new SelectionAdapter() { public void widgetSelected(SelectionEvent event) { // Restoring a backup requires Solr >=5.2 // @see SOLR-6637 InputDialog dialog = new InputDialog(composite.getShell(), "Restore index from a backup", "Note: backup restoration requires Solr >= 5.2.\n\nBackup name (leave empty to pick the latest backup available):", null, null); dialog.open(); if (dialog.getReturnCode() != IDialogConstants.OK_ID) { return; } try { table.restore(dialog.getValue()); } catch (SophieException e) { ExceptionDialog.open(composite.getShell(), e); } } }); // Pack. toolBar.pack(); }
From source file:com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerToolGoLiveTest.java
License:Apache License
private void verifySolrContents() throws Exception { // verify query assertEquals(RECORD_COUNT, executeSolrQuery("*:*").getNumFound()); assertEquals(1, executeSolrQuery("firstname_s:John0001").getNumFound()); int i = 0;/* ww w.j av a 2s. c o m*/ for (SolrDocument doc : executeSolrQuery("*:*")) { assertEquals(String.format("row%04d", i), doc.getFirstValue("id")); assertEquals(String.format("John%04d", i), doc.getFirstValue("firstname_s")); assertEquals(String.format("Doe%04d", i), doc.getFirstValue("lastname_s")); // perform update doc.removeFields("_version_"); SolrInputDocument update = new SolrInputDocument(); for (Map.Entry<String, Object> entry : doc.entrySet()) { update.setField(entry.getKey(), entry.getValue()); } update.setField("firstname_s", String.format("Nadja%04d", i)); COLLECTION1.add(update); i++; } assertEquals(RECORD_COUNT, i); COLLECTION1.commit(); // verify updates assertEquals(RECORD_COUNT, executeSolrQuery("*:*").getNumFound()); i = 0; for (SolrDocument doc : executeSolrQuery("*:*")) { assertEquals(String.format("row%04d", i), doc.getFirstValue("id")); assertEquals(String.format("Nadja%04d", i), doc.getFirstValue("firstname_s")); assertEquals(String.format("Doe%04d", i), doc.getFirstValue("lastname_s")); // perform delete COLLECTION1.deleteById((String) doc.getFirstValue("id")); i++; } assertEquals(RECORD_COUNT, i); COLLECTION1.commit(); // verify deletes assertEquals(0, executeSolrQuery("*:*").size()); }
From source file:com.sp.keyword_generator.SolrManager.java
License:Open Source License
void insert_Keywords(ArrayList<Book> AllBooks) throws SolrServerException, MalformedURLException, IOException { Integer InputListSize;//from w w w . ja v a 2 s. c o m SolrInputDocument InputDoc; int CurrentSolrDocId; int len = AllBooks.size(); ArrayList<SolrInputDocument> InputDocList = new ArrayList<>(); Collection<Object> LastNameArray; SolrDocument CurrentSolrDoc; for (Book ABook : AllBooks) { Iterator<SolrDocument> solrDocIt = getSolrBookIterator(ABook.book_id); while (solrDocIt.hasNext()) { CurrentSolrDoc = solrDocIt.next(); if (CurrentSolrDoc == null) { continue; } CurrentSolrDocId = Integer.parseInt(CurrentSolrDoc.getFieldValue("book_id").toString()); if (ABook.isNotEmpty() && CurrentSolrDocId == ABook.book_id) { LastNameArray = CurrentSolrDoc.getFieldValues("author_lastname"); CurrentSolrDoc.removeFields("keywords"); for (Keyword kw : ABook.keywords) { LOG.debug("Adding " + kw.getConcat()); //if(kw.keyword.equalsIgnoreCase("genre litteraire") || kw.keyword.equalsIgnoreCase("romance")) continue; CurrentSolrDoc.addField("keywords", kw.getConcat()); } //At Last add the author_lastname(s) as keyword with lemma NA and weight : 1 if (LastNameArray != null) { for (Iterator it = LastNameArray.iterator(); it.hasNext();) { CurrentSolrDoc.addField("keywords", ((String) it.next()).toLowerCase() + ".NA|1"); } } //CurrentSolrDoc.setField("searchable", true); InputDoc = ClientUtils.toSolrInputDocument(CurrentSolrDoc); InputDocList.add(InputDoc); InputListSize = InputDocList.size(); if (InputListSize >= COMMIT_SIZE) { solr.add(InputDocList); solr.commit(); InputDocList.clear(); } LOG.info("Book Added To solr Queue " + InputListSize + "/" + COMMIT_SIZE + "/" + len + " : " + ABook.book_id); } } } solr.add(InputDocList); solr.commit(); }
From source file:com.sp.Parser.Utils.java
License:Open Source License
public static SolrDocumentList getTop(SolrDocumentList docs, ArrayList<String> prices, ArrayList<String> filters) { SolrDocumentList SolrDocList = null; try {// w w w . j a v a 2 s . c om SolrDocument PivotDoc = docs.get(0); //store pId book to put it in the index 0 of the final SolrDocList Collection<Object> Top20Product_ids = PivotDoc.getFieldValues("top20"); //store pId book to put it in the index 0 of the final SolrDocList PivotDoc.removeFields("top20"); // get current book top20 with products ids PivotDoc.removeFields("keywords"); Collection<Object> BackupTop20Product_ids = Top20Product_ids; String QueryString = "product_id:("; ArrayList<String> TempQuery = new ArrayList<>(); for (Object product_id : BackupTop20Product_ids) { QueryString += product_id.toString() + " "; TempQuery.add(product_id.toString()); } QueryString += ")"; String prices_fields = ""; for (String string : prices) { prices_fields += string + " "; } SolrQuery Query = new SolrQuery(QueryString); Query.setRows(101); for (String filter : filters) { LOG.info("Top20 Filtering : " + filter); Query.addFilterQuery(filter); } Query.setParam("fl", "product_id book_id author_searchable author_id format_name description title author_firstname" + " file_size publishing_date author_lastname author_rank publisher_id publisher_name" + " permalink nb_pages isbn " + prices_fields); SolrServer solr = new HttpSolrServer(SearchHandler.solr_url); QueryResponse response = solr.query(Query); SolrDocList = response.getResults(); if (!SolrDocList.isEmpty()) { SolrDocList = Utils.SortListByProductId(TempQuery, SolrDocList); if (!SolrDocList.isEmpty()) { SolrDocument temp = SolrDocList.get(0); SolrDocList.set(0, PivotDoc); SolrDocList.add(1, temp); } } else { SolrDocList.add(0, PivotDoc); } } catch (SolrServerException ex) { LOG.info("mmh : ", ex); } LOG.info("SolrDocList Size int getTop : " + SolrDocList.size()); return SolrDocList; }
From source file:it.damore.solr.importexport.App.java
License:Open Source License
/** * @param client/* w w w. j a v a 2 s . com*/ * @param outputFile * @throws SolrServerException * @throws IOException */ private static void readAllDocuments(HttpSolrClient client, File outputFile) throws SolrServerException, IOException { SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery("*:*"); if (config.getFilterQuery() != null) { solrQuery.addFilterQuery(config.getFilterQuery()); } solrQuery.setRows(0); solrQuery.addSort(config.getUniqueKey(), ORDER.asc); // Pay attention to this line String cursorMark = CursorMarkParams.CURSOR_MARK_START; TimeZone.setDefault(TimeZone.getTimeZone("UTC")); // objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true); DateFormat df = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:sss'Z'"); objectMapper.setDateFormat(df); objectMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); QueryResponse r = client.query(solrQuery); long nDocuments = r.getResults().getNumFound(); logger.info("Found " + nDocuments + " documents"); if (!config.getDryRun()) { logger.info("Creating " + config.getFileName()); Set<SkipField> skipFieldsEquals = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.EQUAL).collect(Collectors.toSet()); Set<SkipField> skipFieldsStartWith = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.STARTS_WITH).collect(Collectors.toSet()); Set<SkipField> skipFieldsEndWith = config.getSkipFieldsSet().stream() .filter(s -> s.getMatch() == MatchType.ENDS_WITH).collect(Collectors.toSet()); try (PrintWriter pw = new PrintWriter(outputFile)) { solrQuery.setRows(200); boolean done = false; while (!done) { solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = client.query(solrQuery); String nextCursorMark = rsp.getNextCursorMark(); for (SolrDocument d : rsp.getResults()) { skipFieldsEquals.forEach(f -> d.removeFields(f.getText())); if (skipFieldsStartWith.size() > 0 || skipFieldsEndWith.size() > 0) { Map<String, Object> collect = d.entrySet().stream() .filter(e -> !skipFieldsStartWith.stream() .filter(f -> e.getKey().startsWith(f.getText())).findFirst() .isPresent()) .filter(e -> !skipFieldsEndWith.stream() .filter(f -> e.getKey().endsWith(f.getText())).findFirst().isPresent()) .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); pw.write(objectMapper.writeValueAsString(collect)); } else { pw.write(objectMapper.writeValueAsString(d)); } pw.write("\n"); } if (cursorMark.equals(nextCursorMark)) { done = true; } cursorMark = nextCursorMark; } } } }
From source file:net.yacy.peers.Protocol.java
License:Open Source License
protected static int solrQuery(final SearchEvent event, final SolrQuery solrQuery, final int offset, final int count, final Seed target, final int partitions, final Blacklist blacklist) { if (event.query.getQueryGoal().getQueryString(false) == null || event.query.getQueryGoal().getQueryString(false).length() == 0) { return -1; // we cannot query solr only with word hashes, there is no clear text string }/*from w w w. j a va 2 s . c o m*/ event.addExpectedRemoteReferences(count); if (partitions > 0) solrQuery.set("partitions", partitions); solrQuery.setStart(offset); solrQuery.setRows(count); // set highlighting query attributes if (event.query.contentdom == Classification.ContentDomain.TEXT || event.query.contentdom == Classification.ContentDomain.ALL) { solrQuery.setHighlight(true); solrQuery.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH); //solrQuery.setHighlightRequireFieldMatch(); solrQuery.setHighlightSimplePost("</b>"); solrQuery.setHighlightSimplePre("<b>"); solrQuery.setHighlightSnippets(5); for (CollectionSchema field : snippetFields) solrQuery.addHighlightField(field.getSolrFieldName()); } else { solrQuery.setHighlight(false); } boolean localsearch = target == null || target.equals(event.peers.mySeed()); Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>( event.query.facetfields.size()); Map<String, LinkedHashSet<String>> snippets = new HashMap<String, LinkedHashSet<String>>(); // this will be a list of urlhash-snippet entries final QueryResponse[] rsp = new QueryResponse[] { null }; final SolrDocumentList[] docList = new SolrDocumentList[] { null }; String ip = target.getIP(); {// encapsulate expensive solr QueryResponse object if (localsearch && !Switchboard.getSwitchboard() .getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { // search the local index try { SolrConnector sc = event.getQuery().getSegment().fulltext().getDefaultConnector(); if (!sc.isClosed()) { rsp[0] = sc.getResponseByParams(solrQuery); docList[0] = rsp[0].getResults(); } } catch (final Throwable e) { Network.log.info("SEARCH failed (solr), localpeer (" + e.getMessage() + ")", e); return -1; } } else { try { final boolean myseed = target == event.peers.mySeed(); if (!myseed && !target.getFlagSolrAvailable()) { // skip if peer.dna has flag that last try resulted in error Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); return -1; } final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(ip); final int solrtimeout = Switchboard.getSwitchboard() .getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); Thread remoteRequest = new Thread() { @Override public void run() { this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + target.hash + ")"); try { RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr", solrtimeout); // this is a 'patch configuration' which considers 'solr' as default collection try { SolrConnector solrConnector = new RemoteSolrConnector(instance, myseed ? true : target.getVersion() >= 1.63, "solr"); if (!solrConnector.isClosed()) try { rsp[0] = solrConnector.getResponseByParams(solrQuery); docList[0] = rsp[0].getResults(); } catch (Throwable e) { } finally { solrConnector.close(); } } catch (Throwable ee) { } finally { instance.close(); } } catch (Throwable eee) { } } }; remoteRequest.start(); remoteRequest.join(solrtimeout); // just wait until timeout appears if (remoteRequest.isAlive()) { try { remoteRequest.interrupt(); } catch (Throwable e) { } Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " does not answer (time-out)"); target.setFlagSolrAvailable(false || myseed); return -1; // give up, leave remoteRequest abandoned. } // no need to close this here because that sends a commit to remote solr which is not wanted here } catch (final Throwable e) { Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " (" + e.getMessage() + ")"); target.setFlagSolrAvailable(false || localsearch); return -1; } } if (rsp[0] == null || docList[0] == null) { Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null"); target.setFlagSolrAvailable(false || localsearch); return -1; } // evaluate facets for (String field : event.query.facetfields) { FacetField facet = rsp[0].getFacetField(field); ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator); List<Count> values = facet == null ? null : facet.getValues(); if (values == null) continue; for (Count ff : values) { int c = (int) ff.getCount(); if (c == 0) continue; if (ff.getName().length() == 0) continue; // facet entry without text is not useful result.set(ff.getName(), c); } if (result.size() > 0) facets.put(field, result); } // evaluate snippets Map<String, Map<String, List<String>>> rawsnippets = rsp[0].getHighlighting(); // a map from the urlhash to a map with key=field and value = list of snippets if (rawsnippets != null) { nextsnippet: for (Map.Entry<String, Map<String, List<String>>> re : rawsnippets.entrySet()) { Map<String, List<String>> rs = re.getValue(); for (CollectionSchema field : snippetFields) { if (rs.containsKey(field.getSolrFieldName())) { List<String> s = rs.get(field.getSolrFieldName()); if (s.size() > 0) { LinkedHashSet<String> ls = new LinkedHashSet<String>(); ls.addAll(s); snippets.put(re.getKey(), ls); continue nextsnippet; } } } // no snippet found :( --we don't assign a value here by default; that can be done as an evaluation outside this method } } rsp[0] = null; } // evaluate result if (docList == null || docList[0].size() == 0) { Network.log.info("SEARCH (solr), returned 0 out of 0 documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()); return 0; } List<URIMetadataNode> container = new ArrayList<URIMetadataNode>(); Network.log.info("SEARCH (solr), returned " + docList[0].size() + " out of " + docList[0].getNumFound() + " documents and " + facets.size() + " facets " + facets.keySet().toString() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); int term = count; Collection<SolrInputDocument> docs; if (event.addResultsToLocalIndex) { // only needed to store remote results docs = new ArrayList<SolrInputDocument>(docList[0].size()); } else docs = null; for (final SolrDocument doc : docList[0]) { if (term-- <= 0) { break; // do not process more that requested (in case that evil peers fill us up with rubbish) } // get one single search result if (doc == null) { continue; } URIMetadataNode urlEntry = new URIMetadataNode(doc); if (blacklist.isListed(BlacklistType.SEARCH, urlEntry.url())) { if (Network.log.isInfo()) { if (localsearch) { Network.log.info("local search (solr): filtered blacklisted url " + urlEntry.url()); } else { Network.log.info("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); } } continue; // block with blacklist } final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker .urlInAcceptedDomain(urlEntry.url()); if (urlRejectReason != null) { if (Network.log.isInfo()) { if (localsearch) { Network.log.info("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")"); } else { Network.log.info("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); } } continue; // reject url outside of our domain } // passed all checks, store url if (!localsearch) { // put the remote documents to the local index. We must convert the solr document to a solr input document: if (event.addResultsToLocalIndex) { final SolrInputDocument sid = event.query.getSegment().fulltext().getDefaultConfiguration() .toSolrInputDocument(doc); // the input document stays untouched because it contains top-level cloned objects docs.add(sid); // will be stored to index, and is a full solr document, can be added to firstseen event.query.getSegment().setFirstSeenTime(urlEntry.hash(), Math.min(urlEntry.moddate().getTime(), System.currentTimeMillis())); } // after this conversion we can remove the largest and not used field text_t and synonyms_sxt from the document // because that goes into a search cache and would take a lot of memory in the search cache //doc.removeFields(CollectionSchema.text_t.getSolrFieldName()); doc.removeFields(CollectionSchema.synonyms_sxt.getSolrFieldName()); ResultURLs.stack(ASCII.String(urlEntry.url().hash()), urlEntry.url().getHost(), event.peers.mySeed().hash.getBytes(), UTF8.getBytes(target.hash), EventOrigin.QUERIES); } // add the url entry to the word indexes container.add(urlEntry); } final int dls = docList[0].size(); final int numFound = (int) docList[0].getNumFound(); docList[0].clear(); docList[0] = null; if (localsearch) { event.addNodes(container, facets, snippets, true, "localpeer", numFound); event.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.info( "local search (solr): localpeer sent " + container.size() + "/" + numFound + " references"); } else { if (event.addResultsToLocalIndex) { for (SolrInputDocument doc : docs) { event.query.getSegment().putDocument(doc); } docs.clear(); docs = null; } event.addNodes(container, facets, snippets, false, target.getName() + "/" + target.hash, numFound); event.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.info("remote search (solr): peer " + target.getName() + " sent " + (container.size() == 0 ? 0 : container.size()) + "/" + numFound + " references"); } return dls; }
From source file:org.alfresco.solr.transformer.CachedDocTransformer.java
License:Open Source License
@Override public void transform(SolrDocument doc, int docid, float score) throws IOException { SolrInputDocument cachedDoc = null;//from www .j a va 2 s . c om try { String id = getFieldValueString(doc, FIELD_SOLR4_ID); TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(id); CoreContainer coreContainer = context.getSearcher().getCore().getCoreContainer(); AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer .getMultiCoreHandler(); SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers() .get(context.getSearcher().getCore().getName()); SolrContentStore solrContentStore = srv.getSolrContentStore(); cachedDoc = solrContentStore.retrieveDocFromSolrContentStore(tenantAndDbId.tenant, tenantAndDbId.dbId); } catch (StringIndexOutOfBoundsException e) { // ignore invalid forms .... } if (cachedDoc != null) { Collection<String> fieldNames = cachedDoc.getFieldNames(); for (String fieldName : fieldNames) { SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); if (schemaField != null) { doc.removeFields(fieldName); if (schemaField.multiValued()) { int index = fieldName.lastIndexOf("@{"); if (index == -1) { doc.addField(fieldName, cachedDoc.getFieldValues(fieldName)); } else { String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); Collection<Object> values = cachedDoc.getFieldValues(fieldName); //Guard against null pointer in case data model field name does not match up with cachedDoc field name. if (values != null) { ArrayList<Object> newValues = new ArrayList<Object>(values.size()); for (Object value : values) { if (value instanceof String) { String stringValue = (String) value; int start = stringValue.lastIndexOf('\u0000'); if (start == -1) { newValues.add(stringValue); } else { newValues.add(stringValue.substring(start + 1)); } } else { newValues.add(value); } } doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, newValues); } } } else { int index = fieldName.lastIndexOf("@{"); if (index == -1) { doc.addField(fieldName, cachedDoc.getFieldValue(fieldName)); } else { String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); alfrescoFieldName = alfrescoFieldName.contains(":") ? alfrescoFieldName.replace(":", "_") : alfrescoFieldName; Object value = cachedDoc.getFieldValue(fieldName); if (value instanceof String) { String stringValue = (String) value; int start = stringValue.lastIndexOf('\u0000'); if (start == -1) { doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, stringValue); } else { doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, stringValue.substring(start + 1)); } } else { doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, value); } } } } } } }
From source file:org.alfresco.solr.transformer.DocValueDocTransformer.java
License:Open Source License
@Override public void transform(SolrDocument doc, int docid, float score) throws IOException { for (String fieldName : context.getSearcher().getFieldNames()) { SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); if (schemaField != null) { if (schemaField.hasDocValues()) { SortedDocValues sortedDocValues = context.getSearcher().getSlowAtomicReader() .getSortedDocValues(fieldName); if (sortedDocValues != null) { int ordinal = sortedDocValues.getOrd(docid); if (ordinal > -1) { doc.removeFields(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, schemaField.getType().toObject(schemaField, sortedDocValues.lookupOrd(ordinal))); }/* w w w. j a v a 2 s. c o m*/ } SortedSetDocValues sortedSetDocValues = context.getSearcher().getSlowAtomicReader() .getSortedSetDocValues(fieldName); if (sortedSetDocValues != null) { ArrayList<Object> newValues = new ArrayList<Object>(); sortedSetDocValues.setDocument(docid); long ordinal; while ((ordinal = sortedSetDocValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { newValues.add(schemaField.getType().toObject(schemaField, sortedSetDocValues.lookupOrd(ordinal))); } doc.removeFields(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, newValues); } BinaryDocValues binaryDocValues = context.getSearcher().getSlowAtomicReader() .getBinaryDocValues(fieldName); if (binaryDocValues != null) { doc.removeFields(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); doc.removeFields(alfrescoFieldName); doc.addField(alfrescoFieldName, schemaField.getType().toObject(schemaField, binaryDocValues.get(docid))); } if (schemaField.getType().getNumericType() != null) { NumericDocValues numericDocValues = context.getSearcher().getSlowAtomicReader() .getNumericDocValues(fieldName); if (numericDocValues != null) { doc.removeFields(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); doc.removeFields(alfrescoFieldName); switch (schemaField.getType().getNumericType()) { case DOUBLE: doc.addField(alfrescoFieldName, Double.longBitsToDouble(numericDocValues.get(docid))); break; case FLOAT: doc.addField(alfrescoFieldName, Float.intBitsToFloat((int) numericDocValues.get(docid))); break; case INT: doc.addField(alfrescoFieldName, (int) numericDocValues.get(docid)); break; case LONG: doc.addField(alfrescoFieldName, numericDocValues.get(docid)); break; } } SortedNumericDocValues sortedNumericDocValues = context.getSearcher().getSlowAtomicReader() .getSortedNumericDocValues(fieldName); if (sortedNumericDocValues != null) { sortedNumericDocValues.setDocument(docid); doc.removeFields(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance() .getAlfrescoPropertyFromSchemaField(fieldName); doc.removeFields(alfrescoFieldName); ArrayList<Object> newValues = new ArrayList<Object>(sortedNumericDocValues.count()); if (sortedNumericDocValues.count() > 0) { for (int i = 0; i < sortedNumericDocValues.count(); i++) { switch (schemaField.getType().getNumericType()) { case DOUBLE: newValues.add(NumericUtils .sortableLongToDouble(sortedNumericDocValues.valueAt(i))); break; case FLOAT: newValues.add(NumericUtils .sortableIntToFloat((int) sortedNumericDocValues.valueAt(i))); break; case INT: newValues.add((int) sortedNumericDocValues.valueAt(i)); break; case LONG: newValues.add(sortedNumericDocValues.valueAt(i)); break; } } } doc.addField(alfrescoFieldName, newValues); } } } } } }
From source file:org.apache.nifi.processors.solr.GetSolr.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { final ComponentLog logger = getLogger(); final AtomicBoolean continuePaging = new AtomicBoolean(true); final SolrQuery solrQuery = new SolrQuery(); try {//from w ww. j av a2 s. com if (id_field == null) { id_field = getFieldNameOfUniqueKey(); } final String dateField = context.getProperty(DATE_FIELD).getValue(); final Map<String, String> stateMap = new HashMap<String, String>(); stateMap.putAll(context.getStateManager().getState(Scope.CLUSTER).toMap()); solrQuery.setQuery("*:*"); final String query = context.getProperty(SOLR_QUERY).getValue(); if (!StringUtils.isBlank(query) && !query.equals("*:*")) { solrQuery.addFilterQuery(query); } final StringBuilder automatedFilterQuery = (new StringBuilder()).append(dateField).append(":[") .append(stateMap.get(STATE_MANAGER_FILTER)).append(" TO *]"); solrQuery.addFilterQuery(automatedFilterQuery.toString()); final List<String> fieldList = new ArrayList<String>(); final String returnFields = context.getProperty(RETURN_FIELDS).getValue(); if (!StringUtils.isBlank(returnFields)) { fieldList.addAll(Arrays.asList(returnFields.trim().split("[,]"))); if (!fieldList.contains(dateField)) { fieldList.add(dateField); dateFieldNotInSpecifiedFieldsList.set(true); } for (String returnField : fieldList) { solrQuery.addField(returnField.trim()); } } solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, stateMap.get(STATE_MANAGER_CURSOR_MARK)); solrQuery.setRows(context.getProperty(BATCH_SIZE).asInteger()); final StringBuilder sortClause = (new StringBuilder()).append(dateField).append(" asc,") .append(id_field).append(" asc"); solrQuery.setParam("sort", sortClause.toString()); while (continuePaging.get()) { final QueryRequest req = new QueryRequest(solrQuery); if (isBasicAuthEnabled()) { req.setBasicAuthCredentials(getUsername(), getPassword()); } logger.debug(solrQuery.toQueryString()); final QueryResponse response = req.process(getSolrClient()); final SolrDocumentList documentList = response.getResults(); if (response.getResults().size() > 0) { final SolrDocument lastSolrDocument = documentList.get(response.getResults().size() - 1); final String latestDateValue = df.format(lastSolrDocument.get(dateField)); final String newCursorMark = response.getNextCursorMark(); solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, newCursorMark); stateMap.put(STATE_MANAGER_CURSOR_MARK, newCursorMark); stateMap.put(STATE_MANAGER_FILTER, latestDateValue); FlowFile flowFile = session.create(); flowFile = session.putAttribute(flowFile, "solrQuery", solrQuery.toString()); if (context.getProperty(RETURN_TYPE).getValue().equals(MODE_XML.getValue())) { if (dateFieldNotInSpecifiedFieldsList.get()) { for (SolrDocument doc : response.getResults()) { doc.removeFields(dateField); } } flowFile = session.write(flowFile, SolrUtils.getOutputStreamCallbackToTransformSolrResponseToXml(response)); flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/xml"); } else { final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER) .asControllerService(RecordSetWriterFactory.class); final RecordSchema schema = writerFactory.getSchema(null, null); final RecordSet recordSet = SolrUtils.solrDocumentsToRecordSet(response.getResults(), schema); final StringBuffer mimeType = new StringBuffer(); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { try { final RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, out); writer.write(recordSet); writer.flush(); mimeType.append(writer.getMimeType()); } catch (SchemaNotFoundException e) { throw new ProcessException("Could not parse Solr response", e); } } }); flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeType.toString()); } session.transfer(flowFile, REL_SUCCESS); } continuePaging.set(response.getResults().size() == Integer .parseInt(context.getProperty(BATCH_SIZE).getValue())); } context.getStateManager().setState(stateMap, Scope.CLUSTER); } catch (SolrServerException | SchemaNotFoundException | IOException e) { context.yield(); session.rollback(); logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), e }, e); throw new ProcessException(e); } catch (final Throwable t) { context.yield(); session.rollback(); logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), t }, t); throw t; } }
From source file:org.bibalex.wamcp.storage.WAMCPIndexedStorage.java
License:Apache License
public List<IndexEntry> parseSolrResp(QueryResponse rsp) throws WFSVNException { // if (!this.initialized) { // this.init();// w w w. ja va2 s . c o m // } SolrDocumentList docs = rsp.getResults(); List<IndexEntry> result = new LinkedList<IndexEntry>(); Iterator iter = docs.iterator(); while (iter.hasNext()) { SolrDocument doc = (SolrDocument) iter.next(); IndexEntry entry = new IndexEntry(); entry.name = (String) doc.getFieldValue(WAMCP_INDEX_IDFIELDNAME); // id is the uniqueKey field doc.removeFields(WAMCP_INDEX_IDFIELDNAME); entry.revision = ((Long) doc.getFieldValue(WAMCP_INDEX_FIELDNAME_revision)).longValue(); doc.removeFields(WAMCP_INDEX_FIELDNAME_revision); entry.lastModifiedDate = (Date) doc.getFieldValue(WAMCP_INDEX_FIELDNAME_timestamp); doc.removeFields(WAMCP_INDEX_FIELDNAME_timestamp); entry.lastModifiedBy = (String) doc.getFieldValue(WAMCP_INDEX_lastModifiedByFieldNAME); doc.removeFields(WAMCP_INDEX_lastModifiedByFieldNAME); entry.wfStep = (String) doc.getFieldValue(WAMCP_INDEX_FIELDNAME_WFSTEP); doc.removeFields(WAMCP_INDEX_FIELDNAME_WFSTEP); for (String fldname : doc.getFieldNames()) { String concatedValue = ""; int ix = fldname.indexOf('_'); String fldKey; if (ix != -1) { fldKey = fldname.substring(0, ix); } else { fldKey = fldname; // throw new WFSVNException("Malformed field name: " + fldname); } // String fldType = this.ixedFieldsTypes.get(fldname); if (fldKey.endsWith("SSS")) { // summary field fldKey = fldKey.substring(0, fldKey.length() - 3); } else { // not a summary field continue; } fldKey = fldKey.replace('.', ' '); for (Object obj : doc.getFieldValues(fldname)) { concatedValue += IXED_STORAGE_CONCATED_VALUE_DELIMETIR + obj.toString(); // TODONOT highlighting // if (rsp.getHighlighting().get(entry.name) != null) { // List<String> highlightSnippets = rsp.getHighlighting().get(entry.name).get( // fldname); // } } concatedValue = concatedValue.substring(IXED_STORAGE_CONCATED_VALUE_DELIMETIR .length()); entry.summaryFieldsValues.put(fldKey, concatedValue); } result.add(entry); } return result; }