List of usage examples for org.apache.solr.common SolrInputDocument toString
@Override
public String toString()
From source file:actors.IndexWorker.java
License:Apache License
public void indexingSolr(List<Entry<DbxEntry>> entries) { for (Entry<DbxEntry> entry : entries) { if (entry.metadata == null) { System.out.println("Deleted: " + entry.lcPath); SolrDeleteEvent eventSolr = new SolrDeleteEvent("", entry.lcPath); // logic for delete the doc } else {// w w w . ja v a2 s . c om SolrInputDocument doc = dbhelper.pipeToSolr(entry); if (doc != null) { System.out.println(doc.toString()); } // creating solrinput doc SolrIndexEvent eventSolr = new SolrIndexEvent("", doc); another.tell(eventSolr, getSelf()); } // send to solr using solr index event } }
From source file:com.github.fengtan.sophie.tables.DocumentsTable.java
License:Open Source License
/** * Upload local changes to the Solr server. * //from w w w.j a va 2 s .c o m * @throws SophieException * If the local changes could not be uploaded to the Solr * server. */ public void upload() throws SophieException { // Upload local updates. for (SolrDocument document : documentsUpdated) { SolrInputDocument input = new SolrInputDocument(); for (String name : document.getFieldNames()) { input.addField(name, document.getFieldValue(name), 1.0f); } try { // Returned object seems to have no relevant information. Sophie.client.add(input); } catch (SolrServerException | IOException | SolrException e) { throw new SophieException("Unable to update document in index: " + input.toString(), e); } } // Upload local deletions. for (SolrDocument document : documentsDeleted) { String id = Objects.toString(document.getFieldValue(uniqueField), StringUtils.EMPTY); try { Sophie.client.deleteById(id); } catch (SolrServerException | IOException | SolrException e) { throw new SophieException("Unable to delete document from index: " + id, e); } } // Upload local additions. for (SolrDocument document : documentsAdded) { SolrInputDocument input = new SolrInputDocument(); for (String name : document.getFieldNames()) { input.addField(name, document.getFieldValue(name), BOOST); } try { // Returned object seems to have no relevant information. Sophie.client.add(input); } catch (SolrServerException | IOException | SolrException e) { throw new SophieException("Unable to add document to index: " + input.toString(), e); } } // Commit the index. try { Sophie.client.commit(); } catch (SolrServerException | IOException | SolrException e) { throw new SophieException("Unable to commit index", e); } // Refresh so user can see the new state of the server. refresh(); }
From source file:com.inqool.dcap.office.indexer.indexer.SolrBulkIndexer.java
License:Apache License
public void update(ModelTreeNode data) throws IOException { dataForSuggester.clear();/* ww w. j a va 2s.c om*/ SolrServer server = new HttpSolrServer(SOLR_MAIN_ENDPOINT); fetchOrgNames(); String fedoraId = store.getOnlyIdFromUrl(data.getModel().getUrl()); //First remove old doc try { //first delete children //not needed, if _root_ is indexed, children are deleted with parent final UpdateResponse resp0 = server.deleteByQuery("_root_:\"" + fedoraId + "\""); if (resp0.getStatus() == 0) { logger.debug("Remove request was successful for children of: {}", fedoraId); } else { logger.error("Remove request has error, code: {} for pid: {}", resp0.getStatus(), fedoraId); return; } //then delete the doc itself final UpdateResponse resp1 = server.deleteById(fedoraId); if (resp1.getStatus() == 0) { logger.debug("Remove request was successful for: {}", fedoraId); } else { logger.error("Remove request has error, code: {} for pid: {}", resp1.getStatus(), fedoraId); return; } //Also delete suggester data belonging to the document root SolrServer suggesterServer = new HttpSolrServer(SOLR_SUGGESTER_ENDPOINT); final UpdateResponse resp2 = suggesterServer.deleteByQuery("belongsTo:\"" + fedoraId + "\""); if (resp2.getStatus() == 0) { logger.debug("Remove request was successful for suggester data of: {}", fedoraId); } else { logger.error("Remove request for suggester data has error, code: {} for pid: {}", resp2.getStatus(), fedoraId); return; } } catch (final SolrServerException | IOException e) { logger.error("Delete Exception: {}", e); throw new RuntimeException(e); } //Recursively dig to fedora and triplestore to construct whole solr document hierarchy SolrInputDocument solrInputDocument = recursivelyIndex(data); if (solrInputDocument == null) return; //this was probably a delete request, skip inserting String solrDoc = solrInputDocument.toString(); if (solrDoc.length() > 500) { solrDoc = solrDoc.substring(0, 500); } logger.debug("Created SolrInputDocument: {}", solrDoc); //Then insert new docs try { final UpdateResponse resp = server.add(solrInputDocument); if (resp.getStatus() == 0) { logger.debug("Update request was successful for: {}", fedoraId); } else { logger.error("Update request returned error code: {} for identifier: {}", resp.getStatus(), fedoraId); } logger.debug("Received result from Solr request."); } catch (final SolrServerException | IOException e) { logger.error("Update exception: {}!", e); throw new RuntimeException(e); } //And update also suggester data try { //For data for suggester, add info what root document they belong to, so that they can be also unindexed later with the document for (SolrInputDocument solrInputDoc : dataForSuggester) { solrInputDoc.addField("belongsTo", fedoraId); } SolrServer suggesterServer = new HttpSolrServer(SOLR_SUGGESTER_ENDPOINT); final UpdateResponse resp = suggesterServer.add(dataForSuggester); if (resp.getStatus() == 0) { logger.debug("Updating solr suggester data successful."); } else { logger.error("Update request for solr suggester data returned error code: {} for identifier: {}", resp.getStatus(), fedoraId); } } catch (Exception e) { logger.error("Error updating solr sugesster data.", e); } }
From source file:com.ngdata.hbaseindexer.indexer.FusionDocumentWriter.java
License:Apache License
/** * shs: This method was modified to add the parent and docCount parameters. These were added to support the recursion * needed to flatten (aka denormalize) the parent/child document to any nesting level the input document may have * been nested to.// w w w. j av a2 s.c o m * @param parentSolrDoc This is the parent document of the list of childSolrDocs. It is in SolrInputDocument format. * This parameter will be 'null' when first called. When this method is called recursively, * it will not be null. * @param childSolrDocs This is the list documents that are the children (nested) documents of parentSolrDoc. * @param docCount This is the number of child documents for a parent. It is used as the child ID * @return The list documents to be sent to Fusion for indexing formatted as JSON documents. * @throws Exception */ protected List<Map<String, Object>> toJsonDocs(SolrInputDocument parentSolrDoc, Collection<SolrInputDocument> childSolrDocs, int docCount) throws Exception { boolean isDebugEnabled = log.isDebugEnabled(); if (isDebugEnabled) { log.debug("Method:toJsonDocs - Processing SolrInputDocuments: parent:[" + (parentSolrDoc == null ? "null" : parentSolrDoc.toString()) + "] with " + childSolrDocs.size() + " child documents."); } List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(childSolrDocs.size()); for (SolrInputDocument childSolrDoc : childSolrDocs) { if (isDebugEnabled) { log.debug("Method:toJsonDocs - Processing SolrInputDocuments: parent:[" + (parentSolrDoc == null ? "null" : parentSolrDoc.toString()) + "]; child:[" + childSolrDoc.toString() + "]"); } list.addAll(toJson(parentSolrDoc, childSolrDoc, docCount)); } return list; }
From source file:com.ngdata.hbaseindexer.indexer.FusionDocumentWriter.java
License:Apache License
/** * shs: This method was modified from its original to add the input parameters 'parent' and 'docCount'. This was done * to enable recursion to be used to find all parent/child relationships to any level. The method will merge the * fields in the parent document into the child document and will then convert that merged document into JSON * format and return that JSON document to the caller. * @param parent The parent document for the child document being passed in. Parent may be null if the child being * passed in is a member of the initial documents submitted. * @param child This is the child document. It will have the parent's fields merged into it. * @param docCount This is a count of the number of documents that have been added in this processing. * @return The merged parent and child documents as a JSON formatted document, in a format acceptable to * Fusion.// w w w.j av a2s .c o m */ protected Map<String, Object> doc2json(SolrInputDocument parent, SolrInputDocument child, int docCount) { Map<String, Object> json = new HashMap<String, Object>(); if (child != null) { String docId = (String) child.getFieldValue("id"); if (docId == null) { if (parent != null) { String parentId = (String) parent.getFieldValue("id"); docId = parentId + "-" + docCount; } if (docId == null) throw new IllegalStateException("Couldn't resolve the id for document: " + child); } json.put("id", docId); List fields = new ArrayList(); if (parent != null) { if (log.isDebugEnabled()) log.debug("Method:doc2json - Merging parent and child docs, parent:[" + parent.toString() + "]; child[" + child.toString() + "]."); // have a parent doc ... flatten by adding all parent doc fields to the child with prefix _p_ for (String f : parent.getFieldNames()) { if ("id".equals(f)) { fields.add(mapField("_p_id", null /* field name prefix */, parent.getField("id").getFirstValue())); } else { appendField(parent, f, "_p_", fields); } } } for (String f : child.getFieldNames()) { if (!"id".equals(f)) { // id already added appendField(child, f, null, fields); } } // keep track of the time we saw this doc on the hbase side String tdt = DateUtil.getThreadLocalDateFormat().format(new Date()); fields.add(mapField("_hbasets_tdt", null, tdt)); if (log.isDebugEnabled()) log.debug(strIndexName + " Reconcile id = " + docId + " and timestamp = " + tdt); json.put("fields", fields); } else { log.warn("method:doc2json - Input parameter 'child' was null."); } return json; }
From source file:de.hebis.it.hds.gnd.in.MarcXmlParser.java
License:Open Source License
/** * @param doc// ww w . ja v a 2s. c om */ private boolean checkAndLog(SolrInputDocument doc, String marcXml) { if (LOG.isTraceEnabled()) LOG.trace("New Document: " + doc.toString()); String docId = (String) doc.getFieldValue("id"); if (docId == null) { LOG.error("No Id found in " + marcXml.replace('\n', ' ')); return false; } if (doc.getFieldValue("preferred") == null) { LOG.error(docId + ": No preferred naming found in marcXml. " + marcXml.replace('\n', ' ')); return false; } if (LOG.isDebugEnabled()) { if (doc.getFieldValues("coordinates") != null) { for (Object coordinate : doc.getFieldValues("coordinates")) { LOG.debug(docId + ": Coordinates found [" + (String) coordinate + "]."); } } if (doc.getFieldValue("look4me") == "true") { LOG.debug(docId + ":(" + (String) doc.getFieldValue("preferred") + ") Needs a second pass."); } } int counterNow = counter.getAndIncrement(); if (counterNow % 10000 == 0) LOG.info("Records processed: " + counterNow); return true; }
From source file:edu.cornell.mannlib.vitro.webapp.search.solr.SolrIndexer.java
License:Open Source License
@Override public void index(Individual ind) throws IndexingException { if (!indexing) throw new IndexingException("SolrIndexer: must call " + "startIndexing() before index()."); if (ind == null) { log.debug("in index(), Individual to index was null, ignoring."); return;//from www. j a va 2s. com } try { if (urisIndexed != null && urisIndexed.contains(ind.getURI())) { log.debug("already indexed " + ind.getURI()); return; } else { SolrInputDocument solrDoc = null; synchronized (this) { urisIndexed.add(ind.getURI()); } log.debug("indexing " + ind.getURI()); solrDoc = individualToSolrDoc.translate(ind); if (solrDoc != null) { if (log.isDebugEnabled()) { log.info("boost for " + ind.getName() + " is " + solrDoc.getDocumentBoost()); log.debug(solrDoc.toString()); } UpdateResponse res = server.add(solrDoc); log.debug("response after adding docs to server: " + res); } else { log.debug("removing from index " + ind.getURI()); removeFromIndex(ind.getURI()); } } } catch (IOException ex) { throw new IndexingException(ex.getMessage()); } catch (SolrServerException ex) { throw new IndexingException(ex.getMessage()); } }
From source file:edu.harvard.iq.dataverse.search.SearchUtilTest.java
@Test public void testCreateSolrDoc() { assertEquals(null, SearchUtil.createSolrDoc(null)); Long datasetVersionId = 345678l; SolrInputDocument solrInputDocument = SearchUtil .createSolrDoc(new DvObjectSolrDoc("12345", "dataset_12345", datasetVersionId, "myNameOrTitleNotUsedHere", Arrays.asList(IndexServiceBean.getPublicGroupString()))); System.out.println(solrInputDocument.toString()); assertEquals(// ww w . j a va 2s . c o m SearchFields.ID + "=" + IndexServiceBean.solrDocIdentifierDataset + "12345" + IndexServiceBean.discoverabilityPermissionSuffix, solrInputDocument.get(SearchFields.ID).toString()); assertEquals(SearchFields.DEFINITION_POINT + "=dataset_12345", solrInputDocument.get(SearchFields.DEFINITION_POINT).toString()); assertEquals(SearchFields.DEFINITION_POINT_DVOBJECT_ID + "=12345", solrInputDocument.get(SearchFields.DEFINITION_POINT_DVOBJECT_ID).toString()); assertEquals(SearchFields.DISCOVERABLE_BY + "=" + Arrays.asList(IndexServiceBean.getPublicGroupString()), solrInputDocument.get(SearchFields.DISCOVERABLE_BY).toString()); }
From source file:edu.unc.lib.dl.data.ingest.solr.test.AtomicUpdateTest.java
License:Apache License
@Test public void atomicUpdate() throws IOException { IndexDocumentBean idb = new IndexDocumentBean(); idb.setId("id"); idb.setStatus(Arrays.asList("Unpublished", "Parent Unpublished")); DocumentObjectBinder binder = new DocumentObjectBinder(); SolrInputDocument sid = binder.toSolrInputDocument(idb); String operation = "set"; for (String fieldName : sid.getFieldNames()) { if (!ID_FIELD.equals(fieldName)) { SolrInputField inputField = sid.getField(fieldName); // Adding in each non-null field value, except the timestamp field which gets cleared if not specified so // that it always gets updated as part of a partial update // TODO enable timestamp updating when fix for SOLR-4133 is released, which enables setting null fields if (inputField != null && (inputField.getValue() != null || UPDATE_TIMESTAMP.equals(fieldName))) { Map<String, Object> partialUpdate = new HashMap<String, Object>(); partialUpdate.put(operation, inputField.getValue()); sid.setField(fieldName, partialUpdate); }/*from w w w .ja v a 2 s .c om*/ } } /* Map<String,String> mapField = new HashMap<String,String>(); mapField.put("", arg1) Map<String, Object> partialUpdate = new HashMap<String, Object>(); partialUpdate.put("set", inputField.getFirstValue()); sid.setField(fieldName, partialUpdate);*/ StringWriter writer = new StringWriter(); ClientUtils.writeXML(sid, writer); System.out.println(writer.toString()); System.out.println(sid.toString()); }
From source file:eu.annocultor.data.destinations.SolrServer.java
License:Apache License
private void flush() throws Exception { int docs = documents.size(); System.out.println("Start with " + docs); while (!documents.isEmpty()) { final SolrInputDocument document = documents.pop(); if (document == null || document.getFieldNames().isEmpty()) { docs--;/* www .j a v a 2 s.c om*/ } else { log.info("Document to server: " + document.toString()); try { server.add(document); } catch (Exception e) { e.printStackTrace(); } } } server.commit(); log.info("Committed " + docs + " documents"); }