List of usage examples for org.apache.solr.common SolrDocument getFirstValue
public Object getFirstValue(String name)
From source file:at.newmedialab.lmf.search.services.indexing.SolrCoreRuntime.java
License:Apache License
/** * Ask the server to retrieve all documents that depend on the resource passed as argument; this * query is/* w w w .j a v a2 s .c o m*/ * carried out by querying the dependencies field of a document. * * @param resource * @return */ public Collection<URI> listDependent(ValueFactory valueFactory, URI resource) { SolrQuery query = new SolrQuery(); query.setQuery("lmf.dependencies:\"" + resource.stringValue() + "\""); query.setFields("lmf.uri"); query.setRows(Integer.MAX_VALUE); try { SolrDocumentList docs = server.query(query).getResults(); Set<URI> result = new HashSet<URI>(); for (SolrDocument doc : docs) { result.add(valueFactory.createURI((String) doc.getFirstValue("lmf.uri"))); } return result; } catch (SolrServerException e) { return Collections.emptyList(); } }
From source file:au.org.aekos.shared.api.model.dataset.SharedSearchResultFactory.java
/** * Creates a fully populated new instance from the data in the supplied Solr Document. * // ww w .ja va 2 s .co m * @param solrDoc doc to extract data from * @return new instance of a search result */ public SharedSearchResult newSearchResultInstance(SolrDocument solrDoc) { String resultDatasetId = (String) solrDoc.getFirstValue(idIndexName); String resultTitleDescription = (String) solrDoc.getFirstValue(datasetNameIndexName); String resultAbstractDescription = (String) solrDoc.getFirstValue(datasetAbstractIndexName); String resultThumbnailUrl = (String) solrDoc.getFirstValue(thumbnailUrlIndexName); String resultImageUrl = (String) solrDoc.getFirstValue(imageUrlIndexName); String resultWkt = (String) solrDoc.getFirstValue(spatialLocationIndexName); Date today = new Date(); boolean isEmbargoedToday = determineIsEmbargoedOnDate(solrDoc, today); String indexedLicenseType = (String) solrDoc.getFirstValue(licenseTypeIndexName); String exportedLicenseType = resolveExportLicenseType(indexedLicenseType); SharedSearchResult result = new SharedSearchResult(resultDatasetId, resultThumbnailUrl, resultImageUrl, resultTitleDescription, resultAbstractDescription, resultWkt, isEmbargoedToday, exportedLicenseType); for (GridCellMapper currMapper : gridCellMappings) { if (!currMapper.canExecute(solrDoc)) { continue; } result.addGridField(currMapper.map(solrDoc)); } return result; }
From source file:au.org.aekos.shared.api.model.dataset.SharedSearchResultFactory.java
/** * @param solrDoc document to extract the embargo date from * @param compareToDate compare the embargo date to this date to determine if we are under embargo. * Should be today but can be used against any day. * @return <code>true</code> if we are still under embargo, <code>false</code> otherwise *//* w ww . jav a 2s. c o m*/ boolean determineIsEmbargoedOnDate(SolrDocument solrDoc, Date compareToDate) { if (!solrDoc.containsKey(embargoDateIndexName)) return false; Date embargoDate = (Date) solrDoc.getFirstValue(embargoDateIndexName); return compareToDate.before(embargoDate); }
From source file:au.org.ala.biocache.dao.SearchDAOImpl.java
License:Open Source License
private List<String[]> intersectResults(String layersServiceUrl, String[] analysisLayers, SolrDocumentList results) {/*from ww w .j a v a2 s . c o m*/ List<String[]> intersection = new ArrayList<String[]>(); if (analysisLayers.length > 0 && StringUtils.isNotEmpty(layersServiceUrl)) { try { double[][] points = new double[results.size()][2]; int invalid = 0; int i = 0; for (SolrDocument sd : results) { if (sd.containsKey("sensitive_longitude") && sd.containsKey("sensitive_latitude")) { points[i][0] = (double) sd.getFirstValue("sensitive_longitude"); points[i][1] = (double) sd.getFirstValue("sensitive_latitude"); } else if (sd.containsKey("longitude") && sd.containsKey("latitude")) { points[i][0] = (double) sd.getFirstValue("longitude"); points[i][1] = (double) sd.getFirstValue("latitude"); } else { points[i][0] = 0; points[i][1] = 0; invalid++; } i++; } if (invalid < results.size()) { LayersStore ls = new LayersStore(layersServiceUrl); Reader reader = ls.sample(analysisLayers, points, null); CSVReader csv = new CSVReader(reader); intersection = csv.readAll(); csv.close(); } } catch (IOException e) { logger.error("Failed to intersect analysis layers", e); } } return intersection; }
From source file:com.francelabs.datafari.statistics.StatsProcessor.java
License:Apache License
public static void processStatsResponse(final QueryResponse queryResponse) throws Exception { final NamedList responseHeader = queryResponse.getResponseHeader(); final FacetField QFacet = queryResponse.getFacetField("q"); final Long numTot = queryResponse.getResults().getNumFound(); final SolrDocumentList solrDocumentList = new SolrDocumentList(); solrDocumentList.setNumFound(QFacet.getValueCount()); solrDocumentList.setStart(0);/*from w w w . j a va2 s. c o m*/ if (numTot != 0) { final Map<String, FieldStatsInfo> stats = queryResponse.getFieldStatsInfo(); final List<FieldStatsInfo> noHitsStats = stats.get("noHits").getFacets().get("q"); final List<FieldStatsInfo> QTimeStats = stats.get("QTime").getFacets().get("q"); List<FieldStatsInfo> positionClickTotStats = null; try { positionClickTotStats = stats.get("positionClickTot").getFacets().get("q"); } catch (final Exception e) { } final List<FieldStatsInfo> clickStats = stats.get("click").getFacets().get("q"); final List<FieldStatsInfo> numClicksStats = stats.get("numClicks").getFacets().get("q"); final List<FieldStatsInfo> numFoundStats = stats.get("numFound").getFacets().get("q"); final List<Count> QFacetValues = QFacet.getValues(); final Map<String, SolrDocument> mapDocuments = new HashMap<String, SolrDocument>(); for (int i = 0; i < QFacetValues.size(); i++) { final SolrDocument doc = new SolrDocument(); final String query = QFacetValues.get(i).getName(); final double count = QFacetValues.get(i).getCount(); final double frequency = StatsUtils.round(count * 100 / numTot, 2, BigDecimal.ROUND_HALF_UP); doc.addField("query", query); doc.addField("count", count); doc.addField("frequency", frequency); mapDocuments.put(query, doc); solrDocumentList.add(doc); } for (int i = 0; i < QTimeStats.size(); i++) { final String query = QTimeStats.get(i).getName(); final SolrDocument doc = mapDocuments.get(query); final int AVGHits = new Double((Double) numFoundStats.get(i).getMean()).intValue(); final Double noHits = new Double((Double) noHitsStats.get(i).getSum()); final int AVGQTime = new Double((Double) QTimeStats.get(i).getMean()).intValue(); final int MAXQTime = new Double((Double) QTimeStats.get(i).getMax()).intValue(); final double click = new Double((Double) clickStats.get(i).getSum()); final double clickRatio = StatsUtils.round(click * 100 / (Double) doc.getFirstValue("count"), 2, BigDecimal.ROUND_HALF_UP); if (click > 0) { final double AVGClickPosition = new Double((Double) positionClickTotStats.get(i).getSum() / (Double) numClicksStats.get(i).getSum()).intValue(); doc.addField("AVGClickPosition", AVGClickPosition); } else { doc.addField("AVGClickPosition", "-"); } doc.addField("withClickRatio", clickRatio); doc.addField("AVGHits", AVGHits); doc.addField("numNoHits", noHits); doc.addField("withClick", click); doc.addField("AVGQTime", AVGQTime); doc.addField("MaxQTime", MAXQTime); } } final NamedList<Object> response = new SimpleOrderedMap<Object>(); response.add("responseHeader", responseHeader); response.add("response", solrDocumentList); queryResponse.setResponse(response); }
From source file:com.hurence.logisland.service.solr.SolrTokenizationTest.java
License:Apache License
@Test public void testTokenizerInSolr() throws SolrServerException, IOException { SolrClient server = rule.getClient(); ModifiableSolrParams params = new ModifiableSolrParams(); // ** Let's index a document into our embedded server SolrInputDocument newDoc = new SolrInputDocument(); newDoc.addField("host", "Test Document 1"); newDoc.addField("name", "doc-1"); newDoc.addField("type", "Hello world!"); newDoc.addField("start", new Date().getTime()); newDoc.addField("end", new Date().getTime() + 1000); server.add(newDoc);//from w w w . jav a 2 s .co m server.commit(); // ** And now let's query for it params.set("q", "name:doc-1"); QueryResponse qResp = server.query(params); SolrDocumentList docList = qResp.getResults(); assertTrue(docList.getNumFound() == 1); SolrDocument doc = docList.get(0); assertTrue(doc.getFirstValue("host").equals("Test Document 1")); }
From source file:com.ibm.watson.developer_cloud.retrieve_and_rank.v1.utils.SolrUtils.java
License:Open Source License
/** * Gets the documents by ids./*from w ww .j a v a 2 s .co m*/ * * @param idsToRetrieve the ids of documents to retrieve * @return the documents */ public Map<String, SolrResult> getDocumentsByIds(ArrayList<String> idsToRetrieve) { SolrDocumentList docs; Map<String, SolrResult> idsToDocs = new HashMap<>(); try { docs = solrClient.getById(collectionName, idsToRetrieve, new ModifiableSolrParams()); for (SolrDocument doc : docs) { SolrResult result = new SolrResult(); result.setBody(doc.getFirstValue(BODY).toString().replaceAll("\\s+", " ").trim()); result.setId(doc.getFirstValue(ID).toString()); result.setTitle(doc.getFirstValue(TITLE).toString().replaceAll("\\s+", " ").trim()); idsToDocs.put(result.getId(), result); } } catch (IOException | SolrServerException e) { logger.log(Level.SEVERE, "Error retrieven the Solr documents", e); } return idsToDocs; }
From source file:com.ibm.watson.retrieveandrank.app.rest.RetrieveAndRankProxyResource.java
License:Open Source License
/** * Performs a query against the solr retrieve service and then makes a call to rank the results. The order of the * results after both calls is recorded and the returned results are noted in each payload. Once the ranked results * are retrieved a third API call is made to the solr retrieve service to retrieve the body (text) for each result. * A final lookup is performed to get the ground truth relevance value for each returned result. This final lookup * would not normally be performed, but as a goal of this app is to show the user how training affects the final * results of the ranker, we return that info also. * * @param body/*from w w w .j a v a 2 s . c o m*/ * a query object which contains a textual query, and potentially a query ID to allow us to perform * ground truth lookup. The incoming payload is described by: {@link IncomingQueryPayload} * * @return */ @Path("/query") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @POST public Response postQueryToSolrAndRanker(IncomingQueryPayload body) { final RetrieveAndRankPayload payload = new RetrieveAndRankPayload(); // pay load which will eventually be // returned to client payload.setQuery(body.getQuery()); try { final QueryResponse featureQueryResponse = solrRuntimeQuery(body.getQuery(), true); final int resultSize = featureQueryResponse.getResults().size(); payload.setNum_solr_results(resultSize); // The following call is made to just have a reference set of results. // When we return the results to the client final List<String> finalRank = new ArrayList<>(); Iterator<SolrDocument> it = featureQueryResponse.getResults().iterator(); int i = 0; final ArrayList<RankResultPayload> answerList = new ArrayList<>(); while (it.hasNext()) { final SolrDocument doc = it.next(); final String answerId = (String) doc.getFieldValue(ID_FIELD); finalRank.add(answerId); final String score = String.valueOf(doc.getFieldValue(SCORE_FIELD)); if (i++ < 3) { final RankResultPayload a = new RankResultPayload(); a.setAnswerId((String) doc.getFieldValue(ID_FIELD)); a.setScore(Float.parseFloat(score)); a.setFinalRank(i); if (body.getQueryId() != -1 && groundTruth != null) { // If it is a canned query, get ground truth info if (groundTruth.has(String.valueOf(body.getQueryId()))) { final JsonObject gtForQuery = groundTruth.get(String.valueOf(body.getQueryId())) .getAsJsonObject(); if (gtForQuery.has(a.getAnswerId())) { a.setRelevance(gtForQuery.get(a.getAnswerId()).getAsInt()); } else if (body.getQueryId() != -1) { a.setRelevance(0); } } } answerList.add(a); } } payload.setRanked_results(Lists.newArrayList(answerList)); answerList.clear(); final QueryResponse solrQueryResponse = solrRuntimeQuery(body.getQuery(), false); final List<String> solrRank = new ArrayList<>(); it = solrQueryResponse.getResults().iterator(); i = 0; while (it.hasNext()) { final SolrDocument doc = it.next(); final String answerId = (String) doc.getFieldValue(ID_FIELD); solrRank.add(answerId); final String score = String.valueOf(doc.getFieldValue(SCORE_FIELD)); if (i++ < 3) { final RankResultPayload a = new RankResultPayload(); a.setAnswerId((String) doc.getFieldValue(ID_FIELD)); a.setScore(Float.parseFloat(score)); a.setSolrRank(i); if (body.getQueryId() != -1 && groundTruth != null) { // If it is a canned query, get ground truth info if (groundTruth.has(String.valueOf(body.getQueryId()))) { final JsonObject gtForQuery = groundTruth.get(String.valueOf(body.getQueryId())) .getAsJsonObject(); if (gtForQuery.has(a.getAnswerId())) { a.setRelevance(gtForQuery.get(a.getAnswerId()).getAsInt()); } else if (body.getQueryId() != -1) { a.setRelevance(0); } } } answerList.add(a); } } payload.setSolr_results(answerList); final ArrayList<String> idsOfDocsToRetrieve = new ArrayList<>(); // We only deal with the first three solr and rank results above.. // We need to go through all results and add the solr/rank position // This allows us to show the position of the result in the opposing search.. // For instance we can say result X in the ranked results was at position k in the solr results for (final RankResultPayload answer : payload.getRanked_results()) { idsOfDocsToRetrieve.add(answer.getAnswerId()); answer.setSolrRank(solrRank.indexOf(answer.getAnswerId()));// add 1 as we don't want - } for (final RankResultPayload answer : payload.getSolr_results()) { idsOfDocsToRetrieve.add(answer.getAnswerId()); answer.setFinalRank(finalRank.indexOf(answer.getAnswerId())); // add 1 so we don't end up with zero // index in ui } final ModifiableSolrParams params = new ModifiableSolrParams(); final SolrDocumentList docs = solrClient.getById(COLLECTION_NAME, idsOfDocsToRetrieve, params); it = docs.iterator(); final HashMap<String, SolrResult> idsToDocs = new HashMap<>(); while (it.hasNext()) { // get the full search results final SolrDocument doc = it.next(); final SolrResult result = new SolrResult(); result.body = doc.getFirstValue("body").toString().trim(); result.id = doc.getFirstValue("id").toString(); result.title = doc.getFirstValue("title").toString().trim(); idsToDocs.put(result.id, result); } // Update the solr and rank results with full info for (final RankResultPayload answer : payload.getRanked_results()) { answer.setBody(idsToDocs.get(answer.getAnswerId()).body); answer.setTitle(idsToDocs.get(answer.getAnswerId()).title); } for (final RankResultPayload answer : payload.getSolr_results()) { answer.setBody(idsToDocs.get(answer.getAnswerId()).body); answer.setTitle(idsToDocs.get(answer.getAnswerId()).title); } } catch (final IOException e) { final String message = Messages.getString("RetrieveAndRankProxyResource.RNR_IO_EXCEPTION_IN_QUERY"); final ServerErrorPayload error = new ServerErrorPayload(message); UtilityFunctions.logger.error(message, e); return Response.serverError().entity(error).build(); } catch (final SolrServerException e) { final String message = Messages.getString("RetrieveAndRankProxyResource.RNR_SOLR_EXCEPTION_IN_QUERY"); final ServerErrorPayload error = new ServerErrorPayload(message); UtilityFunctions.logger.error(message, e); return Response.serverError().entity(error).build(); } catch (final InterruptedException e) { final String message = Messages.getString("RetrieveAndRankProxyResource.RNR_PROCESS_QUERY_IN_QUERY"); final ServerErrorPayload error = new ServerErrorPayload(message); UtilityFunctions.logger.error(message, e); return Response.serverError().entity(error).build(); } return Response.ok(payload).build(); }
From source file:com.ngdata.hbaseindexer.indexer.IndexerIT.java
License:Apache License
@Test public void testCustomKeyFormatter() throws Exception { createTable("table1", "family1"); HTable table = new HTable(conf, "table1"); StringBuilder indexerConf = new StringBuilder(); indexerConf.append("<indexer table='table1'"); indexerConf.append(//w w w . ja v a2 s .c o m " unique-key-formatter='com.ngdata.hbaseindexer.uniquekey.HexUniqueKeyFormatter'>"); indexerConf.append(" <field name='field1_s' value='family1:field1' type='string'/>"); indexerConf.append("</indexer>"); createIndexer1(indexerConf.toString()); SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1")); Put put = new Put(new byte[] { 0, 0, 0, 0 }); put.add(b("family1"), b("field1"), b("value1")); table.put(put); SepTestUtil.waitOnReplication(conf, 60000L); collection1.commit(); QueryResponse response = collection1.query(new SolrQuery("*:*")); assertEquals(1, response.getResults().size()); SolrDocument doc = response.getResults().get(0); assertEquals("00000000", doc.getFirstValue("id").toString()); table.close(); }
From source file:com.ngdata.hbaseindexer.indexer.IndexerIT.java
License:Apache License
@Test public void testDefaultKeyFormatter() throws Exception { createTable("table1", "family1"); HTable table = new HTable(conf, "table1"); StringBuilder indexerConf = new StringBuilder(); indexerConf.append("<indexer table='table1'>"); indexerConf.append(" <field name='field1_s' value='family1:field1' type='string'/>"); indexerConf.append("</indexer>"); createIndexer1(indexerConf.toString()); SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1")); Put put = new Put(new byte[] { 0, 0, 0, 0 }); put.add(b("family1"), b("field1"), b("value1")); table.put(put);/* ww w . jav a2 s . c o m*/ SepTestUtil.waitOnReplication(conf, 60000L); collection1.commit(); QueryResponse response = collection1.query(new SolrQuery("*:*")); assertEquals(1, response.getResults().size()); SolrDocument doc = response.getResults().get(0); assertEquals("#0;#0;#0;#0;", doc.getFirstValue("id").toString()); table.close(); }