List of usage examples for org.apache.solr.common SolrDocument getFieldValues
@SuppressWarnings("unchecked") @Override public Collection<Object> getFieldValues(String name)
From source file:SolrUpdate.java
License:Apache License
public void getAllSolrRecords() { String pmid;// ww w . ja v a 2 s . co m try { CoreAdminRequest adminRequest = new CoreAdminRequest(); adminRequest.setAction(CoreAdminAction.RELOAD); SolrServer solr = new HttpSolrServer("http://localhost:8983/solr"); String query; query = "pmid:*"; SolrQuery theq = new SolrQuery(); theq.setQuery(query); theq.setStart(0); theq.setRows(10000); QueryResponse response = new QueryResponse(); response = solr.query(theq); SolrDocumentList list = response.getResults(); int docnum = 1; for (SolrDocument doc : list) { Publication currlist = new Publication(); List<String> fullnames = new ArrayList<String>(); String currepubsum1 = "", currepubsum2 = ""; if (doc.getFieldValue("abstract") != null) { currlist.setAbstract(doc.getFieldValue("abstract").toString()); } if (doc.getFieldValue("ptitle") != null) { currlist.setTitle(doc.getFieldValue("ptitle").toString()); } if (doc.getFieldValue("author_fullname_list") != null) { currlist.setFirst5authors(doc.getFieldValue("author_fullname_list").toString()); } if (doc.getFieldValue("pmid") != null) { currlist.setPmid(Integer.valueOf(doc.getFieldValue("pmid").toString())); } if (doc.getFieldValue("completion") != null) { currlist.setCompletion(Boolean.valueOf(doc.getFieldValue("completion").toString())); } if (doc.getFieldValue("lruid") != null) { currlist.setLruid(doc.getFieldValue("lruid").toString()); } if (doc.getFieldValue("draftpoint") != null) { currlist.setDraftpoint(Integer.valueOf(doc.getFieldValue("draftpoint").toString())); } if (doc.getFieldValue("journalname") != null) { currlist.setJournalname(doc.getFieldValue("journalname").toString()); } if (doc.getFieldValue("journalyear") != null) { currlist.setJournalyear(doc.getFieldValue("journalyear").toString()); } if (doc.getFieldValue("journalday") != null) { currlist.setJournalday(doc.getFieldValue("journalday").toString()); } if (doc.getFieldValue("journalmonth") != null) { currlist.setJournalmonth(doc.getFieldValue("journalmonth").toString()); } if (doc.getFieldValue("journalpage") != null) { currlist.setJournalstartpg(doc.getFieldValue("journalpage").toString()); } if (doc.getFieldValue("journalissue") != null) { currlist.setJournalissue(doc.getFieldValue("journalissue").toString()); } if (doc.getFieldValue("journalvolume") != null) { currlist.setJournalvolume(doc.getFieldValue("journalvolume").toString()); } if (doc.getFieldValue("publicationdate_year") != null) { currlist.setYear(doc.getFieldValue("publicationdate_year").toString()); } if (doc.getFieldValue("doi") != null) { currlist.setDoi(doc.getFieldValue("doi").toString()); } if (doc.getFieldValues("pfileinfo") != null) { Collection<Object> currcoll = doc.getFieldValues("pfileinfo"); for (Object currobj : currcoll) { currlist.getFilesanddata().add(currobj.toString()); } } if (doc.getFieldValue("author_firstname") != null) { currlist.setFauthors((List<String>) doc.getFieldValue("author_firstname")); } if (doc.getFieldValue("author_lastname") != null) { currlist.setLauthors((List<String>) doc.getFieldValue("author_lastname")); } if (doc.getFieldValue("epubmonth") != null) { currlist.setEpubmonth(doc.getFieldValue("epubmonth").toString()); } if (doc.getFieldValue("epubyear") != null) { currlist.setEpubyear(doc.getFieldValue("epubyear").toString()); } if (doc.getFieldValue("epubday") != null) { currlist.setEpubday(doc.getFieldValue("epubday").toString()); } int counter = 0; for (String currstring : currlist.getFauthors()) { currstring += " " + currlist.getLauthors().get(counter); fullnames.add(currstring); counter++; } currlist.setFullnames(fullnames); if (currlist.getJournalvolume().length() > 0) { currepubsum2 += currlist.getJournalvolume(); } if (currlist.getJournalissue().length() > 0) { currepubsum2 += "(" + currlist.getJournalissue() + ")" + ":"; } if (currlist.getJournalstartpg().length() > 0) { currepubsum2 += currlist.getJournalstartpg() + "."; } if (currlist.getEpubday().length() < 1 && currlist.getEpubmonth().length() < 1 && currlist.getEpubyear().length() < 1) { currepubsum1 = "[Epub ahead of print]"; } else if (currlist.getEpubyear().length() > 0) { currepubsum1 = "Epub " + currlist.getEpubyear() + " " + currlist.getEpubmonth() + " " + currlist.getEpubday(); } else { currepubsum1 = ""; } currlist.setEpubsum(currepubsum1); currlist.setEpubsum2(currepubsum2); currlist.setIndex(docnum); if (currlist.getCompletion() == false) { currlist.setComp("Hidden"); } else { currlist.setComp("Visible"); } solrrecords.add(currlist); docnum++; } } catch (Exception ex) { System.out.println(ex); } System.out.println("There are a total of this many records gathered: " + solrrecords.size()); }
From source file:au.org.ala.biocache.dao.SearchDAOImpl.java
License:Open Source License
private int processQueryResults(ConcurrentMap<String, AtomicInteger> uidStats, String[] fields, String[] qaFields, RecordWriter rw, QueryResponse qr, DownloadDetailsDTO dd, boolean checkLimit, AtomicInteger resultsCount, long maxDownloadSize, String[] analysisLayers) { //handle analysis layer intersections List<String[]> intersection = intersectResults(dd.getRequestParams().getLayersServiceUrl(), analysisLayers, qr.getResults());//from w ww .j ava2 s .c o m int count = 0; int record = 0; for (SolrDocument sd : qr.getResults()) { if (sd.getFieldValue("data_resource_uid") != null && (!checkLimit || (checkLimit && resultsCount.intValue() < maxDownloadSize))) { //resultsCount++; count++; resultsCount.incrementAndGet(); //add the record String[] values = new String[fields.length + analysisLayers.length + qaFields.length]; //get all the "single" values from the index for (int j = 0; j < fields.length; j++) { Collection<Object> allValues = sd.getFieldValues(fields[j]); if (allValues == null) { values[j] = ""; } else { Iterator it = allValues.iterator(); while (it.hasNext()) { Object value = it.next(); if (values[j] != null && values[j].length() > 0) values[j] += "|"; //multivalue separator if (value instanceof Date) { values[j] = value == null ? "" : org.apache.commons.lang.time.DateFormatUtils.format((Date) value, "yyyy-MM-dd"); } else { values[j] = value == null ? "" : value.toString(); } //allow requests to include multiple values when requested if (dd == null || dd.getRequestParams() == null || dd.getRequestParams().getIncludeMultivalues() == null || !dd.getRequestParams().getIncludeMultivalues()) { break; } } } } //add analysis layer intersections if (analysisLayers.length > 0 && intersection.size() > record) { //+1 offset for header in intersection list String[] sampling = intersection.get(record + 1); //+2 offset for latitude,longitude columns in sampling array if (sampling != null && sampling.length == analysisLayers.length + 2) { System.arraycopy(sampling, 2, values, fields.length, sampling.length - 2); } } //now handle the assertions java.util.Collection<Object> assertions = sd.getFieldValues("assertions"); //Handle the case where there a no assertions against a record if (assertions == null) { assertions = Collections.EMPTY_LIST; } for (int k = 0; k < qaFields.length; k++) { values[fields.length + k] = Boolean.toString(assertions.contains(qaFields[k])); } rw.write(values); //increment the counters.... incrementCount(uidStats, sd.getFieldValue("institution_uid")); incrementCount(uidStats, sd.getFieldValue("collection_uid")); incrementCount(uidStats, sd.getFieldValue("data_provider_uid")); incrementCount(uidStats, sd.getFieldValue("data_resource_uid")); } record++; } dd.updateCounts(count); return count; }
From source file:com.cloudera.cdk.morphline.solr.AbstractSolrMorphlineTest.java
License:Apache License
protected void testDocumentContent(HashMap<String, ExpectedResult> expectedResultMap) throws Exception { QueryResponse rsp = solrServer.query(new SolrQuery("*:*").setRows(Integer.MAX_VALUE)); // Check that every expected field/values shows up in the actual query for (Entry<String, ExpectedResult> current : expectedResultMap.entrySet()) { String field = current.getKey(); for (String expectedFieldValue : current.getValue().getFieldValues()) { ExpectedResult.CompareType compareType = current.getValue().getCompareType(); boolean foundField = false; for (SolrDocument doc : rsp.getResults()) { Collection<Object> actualFieldValues = doc.getFieldValues(field); if (compareType == ExpectedResult.CompareType.equals) { if (actualFieldValues != null && actualFieldValues.contains(expectedFieldValue)) { foundField = true; break; }//from w w w . j ava2 s . c o m } else { for (Iterator<Object> it = actualFieldValues.iterator(); it.hasNext();) { String actualValue = it.next().toString(); // test only supports string comparison if (actualFieldValues != null && actualValue.contains(expectedFieldValue)) { foundField = true; break; } } } } assert (foundField); // didn't find expected field/value in query } } }
From source file:com.cloudera.cdk.morphline.solr.AbstractSolrMorphlineZkTest.java
License:Apache License
private Record toRecord(SolrDocument doc) { Record record = new Record(); for (String key : doc.keySet()) { record.getFields().replaceValues(key, doc.getFieldValues(key)); }/* ww w. j a v a 2 s . co m*/ return record; }
From source file:com.digitalpebble.stormcrawler.solr.persistence.SolrSpout.java
License:Apache License
private void populateBuffer() { // TODO Sames as the ElasticSearchSpout? // TODO Use the cursor feature? // https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results SolrQuery query = new SolrQuery(); query.setQuery("*:*").addFilterQuery("nextFetchDate:[* TO NOW]").setStart(lastStartOffset) .setRows(this.bufferSize); if (StringUtils.isNotBlank(diversityField)) { query.addFilterQuery(String.format("{!collapse field=%s}", diversityField)); query.set("expand", "true").set("expand.rows", diversityBucketSize); }// w w w . j a v a 2s . co m try { QueryResponse response = connection.getClient().query(query); SolrDocumentList docs = new SolrDocumentList(); if (StringUtils.isNotBlank(diversityField)) { // Add the main documents collapsed by the CollapsingQParser // plugin docs.addAll(response.getResults()); Map<String, SolrDocumentList> expandedResults = response.getExpandedResults(); for (String key : expandedResults.keySet()) { docs.addAll(expandedResults.get(key)); } } else { docs = response.getResults(); } int numhits = response.getResults().size(); // no more results? if (numhits == 0) lastStartOffset = 0; else lastStartOffset += numhits; String prefix = mdPrefix.concat("."); for (SolrDocument doc : docs) { String url = (String) doc.get("url"); // is already being processed - skip it! if (beingProcessed.containsKey(url)) continue; Metadata metadata = new Metadata(); Iterator<String> keyIterators = doc.getFieldNames().iterator(); while (keyIterators.hasNext()) { String key = keyIterators.next(); if (key.startsWith(prefix)) { Collection<Object> values = doc.getFieldValues(key); key = StringUtils.replace(key, prefix, "", 1); Iterator<Object> valueIterator = values.iterator(); while (valueIterator.hasNext()) { String value = (String) valueIterator.next(); metadata.addValue(key, value); } } } buffer.add(new Values(url, metadata)); } } catch (Exception e) { LOG.error("Can't query Solr: {}", e); } }
From source file:com.doculibre.constellio.utils.NamedListUtils.java
License:Open Source License
private static Element newSolrDocument(SolrDocument doc) { Element nlElement = new Element("lst"); // Unit test prefer when field are sorted List<String> keys = new ArrayList<String>(); keys.addAll(doc.getFieldNames());/* ww w . ja v a 2 s .c o m*/ Collections.sort(keys); for (String key : keys) { Collection<Object> c = doc.getFieldValues(key); Object value = c.size() > 1 ? c : c.iterator().next(); if (value != null) { Element child = toElement(value); if (child != null) { nameElement(child, key); nlElement.addContent(child); } } } return nlElement; }
From source file:com.doculibre.constellio.wicket.panels.results.DefaultSearchResultPanel.java
License:Open Source License
public DefaultSearchResultPanel(String id, SolrDocument doc, final SearchResultsDataProvider dataProvider) { super(id);/*from w w w .ja v a 2s . co m*/ RecordCollectionServices collectionServices = ConstellioSpringUtils.getRecordCollectionServices(); RecordServices recordServices = ConstellioSpringUtils.getRecordServices(); SearchInterfaceConfigServices searchInterfaceConfigServices = ConstellioSpringUtils .getSearchInterfaceConfigServices(); String collectionName = dataProvider.getSimpleSearch().getCollectionName(); RecordCollection collection = collectionServices.get(collectionName); Record record = recordServices.get(doc); if (record != null) { SearchInterfaceConfig searchInterfaceConfig = searchInterfaceConfigServices.get(); IndexField uniqueKeyField = collection.getUniqueKeyIndexField(); IndexField defaultSearchField = collection.getDefaultSearchIndexField(); IndexField urlField = collection.getUrlIndexField(); IndexField titleField = collection.getTitleIndexField(); if (urlField == null) { urlField = uniqueKeyField; } if (titleField == null) { titleField = urlField; } final String recordURL = record.getUrl(); final String displayURL; if (record.getDisplayUrl().startsWith("/get?file=")) { HttpServletRequest req = ((WebRequest) getRequest()).getHttpServletRequest(); displayURL = ContextUrlUtils.getContextUrl(req) + record.getDisplayUrl(); } else { displayURL = record.getDisplayUrl(); } String title = record.getDisplayTitle(); final String protocol = StringUtils.substringBefore(displayURL, ":"); boolean linkEnabled = isLinkEnabled(protocol); // rcupration des champs highlight partir de la cl unique // du document, dans le cas de Nutch c'est l'URL QueryResponse response = dataProvider.getQueryResponse(); Map<String, Map<String, List<String>>> highlighting = response.getHighlighting(); Map<String, List<String>> fieldsHighlighting = highlighting.get(recordURL); String titleHighlight = getTitleFromHighlight(titleField.getName(), fieldsHighlighting); if (titleHighlight != null) { title = titleHighlight; } String excerpt = null; String description = getDescription(record); String summary = getSummary(record); if (StringUtils.isNotBlank(description) && searchInterfaceConfig.isDescriptionAsExcerpt()) { excerpt = description; } else { excerpt = getExcerptFromHighlight(defaultSearchField.getName(), fieldsHighlighting); if (excerpt == null) { excerpt = description; } } toggleSummaryLink = new WebMarkupContainer("toggleSummaryLink"); add(toggleSummaryLink); toggleSummaryLink.setVisible(StringUtils.isNotBlank(summary)); toggleSummaryLink.add(new AttributeModifier("onclick", new LoadableDetachableModel() { @Override protected Object load() { return "toggleSearchResultSummary('" + summaryLabel.getMarkupId() + "')"; } })); summaryLabel = new Label("summary", summary); add(summaryLabel); summaryLabel.setOutputMarkupId(true); summaryLabel.setVisible(StringUtils.isNotBlank(summary)); ExternalLink titleLink; if (displayURL.startsWith("file://")) { HttpServletRequest req = ((WebRequest) getRequest()).getHttpServletRequest(); String newDisplayURL = ContextUrlUtils.getContextUrl(req) + "app/getSmbFile?" + SmbServletPage.RECORD_ID + "=" + record.getId() + "&" + SmbServletPage.COLLECTION + "=" + collectionName; titleLink = new ExternalLink("titleLink", newDisplayURL); } else { titleLink = new ExternalLink("titleLink", displayURL); } final RecordModel recordModel = new RecordModel(record); AttributeModifier computeClickAttributeModifier = new AttributeModifier("onmousedown", true, new LoadableDetachableModel() { @Override protected Object load() { Record record = recordModel.getObject(); SimpleSearch simpleSearch = dataProvider.getSimpleSearch(); WebRequest webRequest = (WebRequest) RequestCycle.get().getRequest(); HttpServletRequest httpRequest = webRequest.getHttpServletRequest(); return ComputeSearchResultClickServlet.getCallbackJavascript(httpRequest, simpleSearch, record); } }); titleLink.add(computeClickAttributeModifier); titleLink.setEnabled(linkEnabled); boolean resultsInNewWindow; PageParameters params = RequestCycle.get().getPageParameters(); if (params != null && params.getString(POPUP_LINK) != null) { resultsInNewWindow = params.getBoolean(POPUP_LINK); } else { resultsInNewWindow = searchInterfaceConfig.isResultsInNewWindow(); } titleLink.add(new SimpleAttributeModifier("target", resultsInNewWindow ? "_blank" : "_self")); // Add title title = StringUtils.remove(title, "\n"); title = StringUtils.remove(title, "\r"); if (StringUtils.isEmpty(title)) { title = StringUtils.defaultString(displayURL); title = StringUtils.substringAfterLast(title, "/"); title = StringUtils.substringBefore(title, "?"); try { title = URLDecoder.decode(title, "UTF-8"); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block e.printStackTrace(); } if (title.length() > 120) { title = title.substring(0, 120) + " ..."; } } Label titleLabel = new Label("title", title); titleLink.add(titleLabel.setEscapeModelStrings(false)); add(titleLink); Label excerptLabel = new Label("excerpt", excerpt); add(excerptLabel.setEscapeModelStrings(false)); // add(new ExternalLink("url", url, // url).add(computeClickAttributeModifier).setEnabled(linkEnabled)); if (displayURL.startsWith("file://")) { // Creates a Windows path for file URLs String urlLabel = StringUtils.substringAfter(displayURL, "file:"); urlLabel = StringUtils.stripStart(urlLabel, "/"); urlLabel = "\\\\" + StringUtils.replace(urlLabel, "/", "\\"); try { urlLabel = URLDecoder.decode(urlLabel, "UTF-8"); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block e.printStackTrace(); } add(new Label("url", urlLabel)); } else { add(new Label("url", displayURL)); } final ReloadableEntityModel<RecordCollection> collectionModel = new ReloadableEntityModel<RecordCollection>( collection); add(new ListView("searchResultFields", new LoadableDetachableModel() { @Override protected Object load() { RecordCollection collection = collectionModel.getObject(); return collection.getSearchResultFields(); } /** * Detaches from the current request. Implement this method with * custom behavior, such as setting the model object to null. */ protected void onDetach() { recordModel.detach(); collectionModel.detach(); } }) { @Override protected void populateItem(ListItem item) { SearchResultField searchResultField = (SearchResultField) item.getModelObject(); IndexFieldServices indexFieldServices = ConstellioSpringUtils.getIndexFieldServices(); Record record = recordModel.getObject(); IndexField indexField = searchResultField.getIndexField(); Locale locale = getLocale(); String indexFieldTitle = indexField.getTitle(locale); if (StringUtils.isBlank(indexFieldTitle)) { indexFieldTitle = indexField.getName(); } StringBuffer fieldValueSb = new StringBuffer(); List<Object> fieldValues = indexFieldServices.extractFieldValues(record, indexField); Map<String, String> defaultLabelledValues = indexFieldServices .getDefaultLabelledValues(indexField, locale); for (Object fieldValue : fieldValues) { if (fieldValueSb.length() > 0) { fieldValueSb.append("\n"); } String fieldValueLabel = indexField.getLabelledValue("" + fieldValue, locale); if (fieldValueLabel == null) { fieldValueLabel = defaultLabelledValues.get("" + fieldValue); } if (fieldValueLabel == null) { fieldValueLabel = "" + fieldValue; } fieldValueSb.append(fieldValueLabel); } item.add(new Label("indexField", indexFieldTitle)); item.add(new MultiLineLabel("indexFieldValue", fieldValueSb.toString())); item.setVisible(fieldValueSb.length() > 0); } @SuppressWarnings("unchecked") @Override public boolean isVisible() { boolean visible = super.isVisible(); if (visible) { List<SearchResultField> searchResultFields = (List<SearchResultField>) getModelObject(); visible = !searchResultFields.isEmpty(); } return visible; } }); // md5 ConstellioSession session = ConstellioSession.get(); ConstellioUser user = session.getUser(); // TODO Provide access to unauthenticated users ? String md5 = ""; if (user != null) { IntelliGIDServiceInfo intelligidServiceInfo = ConstellioSpringUtils.getIntelliGIDServiceInfo(); if (intelligidServiceInfo != null) { Collection<Object> md5Coll = doc.getFieldValues(IndexField.MD5); if (md5Coll != null) { for (Object md5Obj : md5Coll) { try { String md5Str = new String(Hex.encodeHex(Base64.decodeBase64(md5Obj.toString()))); InputStream is = HttpClientHelper .get(intelligidServiceInfo.getIntelligidUrl() + "/connector/checksum", "md5=" + URLEncoder.encode(md5Str, "ISO-8859-1"), "username=" + URLEncoder.encode(user.getUsername(), "ISO-8859-1"), "password=" + URLEncoder.encode(Base64.encodeBase64String( ConstellioSession.get().getPassword().getBytes())), "ISO-8859-1"); try { Document xmlDocument = new SAXReader().read(is); Element root = xmlDocument.getRootElement(); for (Iterator<Element> it = root.elementIterator("fichier"); it.hasNext();) { Element fichier = it.next(); String url = fichier.attributeValue("url"); md5 += "<a href=\"" + url + "\">" + url + "</a> "; } } finally { IOUtils.closeQuietly(is); } } catch (Exception e) { e.printStackTrace(); } } } } } Label md5Label = new Label("md5", md5) { public boolean isVisible() { boolean visible = super.isVisible(); if (visible) { visible = StringUtils.isNotBlank(this.getModelObjectAsString()); } return visible; } }; md5Label.setEscapeModelStrings(false); add(md5Label); add(new ElevatePanel("elevatePanel", record, dataProvider.getSimpleSearch())); } else { setVisible(false); } }
From source file:com.doculibre.constellio.wicket.panels.results.PopupSearchResultPanel.java
License:Open Source License
/** * Will never return null// w w w. j a va 2 s .c o m * * @param doc * @param fieldName * @return */ private String getFieldValue(SolrDocument doc, String fieldName) { Collection<Object> values = doc.getFieldValues(fieldName); if (values == null) { return ""; } return StringUtils.join(values.toArray(), ", "); }
From source file:com.gisgraphy.domain.valueobject.SolrResponseDto.java
License:Open Source License
private Map<String, List<String>> getFieldsToMap(SolrDocument solrDocument, String fieldNamePrefix) { Map<String, List<String>> result = new HashMap<String, List<String>>(); for (String fieldName : solrDocument.getFieldNames()) { if (fieldName.startsWith(fieldNamePrefix)) { for (Object fieldValue : solrDocument.getFieldValues(fieldName)) { String fieldValueString = (String) fieldValue; String languageCode = fieldName.substring(fieldName.lastIndexOf("_") + 1); List<String> languageList = result.get(languageCode); if (languageList == null) { languageList = new ArrayList<String>(); result.put(languageCode, languageList); }// www. jav a 2 s . c om languageList.add(fieldValueString); } } } return result; }
From source file:com.gisgraphy.domain.valueobject.SolrResponseDto.java
License:Open Source License
private List<String> getFieldsToList(SolrDocument solrDocument, String fieldname) { List<String> list = new ArrayList<String>(); if (solrDocument.getFieldValues(fieldname) != null) { for (Object o : solrDocument.getFieldValues(fieldname)) { if (o == null) { continue; } else if (o instanceof String) { list.add(o.toString());/*from w w w. ja v a 2 s . c om*/ } else { throw new RepositoryException( fieldname + " is not a String but a " + o.getClass().getSimpleName()); } } } return list; }