List of usage examples for org.apache.solr.client.solrj.request UpdateRequest setParams
public void setParams(ModifiableSolrParams params)
From source file:com.digitalpebble.behemoth.solr.SOLRWriter.java
License:Apache License
public void write(BehemothDocument doc) throws IOException { final SolrInputDocument inputDoc = convertToSOLR(doc); try {/*from ww w . j a v a 2s. c o m*/ progress.progress(); if (params == null) { solr.add(inputDoc); } else { UpdateRequest req = new UpdateRequest(); req.setParams(params); req.add(inputDoc); solr.request(req); } } catch (SolrServerException e) { throw makeIOException(e); } }
From source file:edu.harvard.gis.hhypermap.bop.SolrUpdateSlammer.java
License:Apache License
private void sendDoc(SolrInputDocument doc) throws IOException, SolrServerException { UpdateRequest req = new UpdateRequest(); req.add(doc);// w ww . j av a2 s . c o m req.setParams(updateParams); req.setCommitWithin(1000); req.process(client, collection); }
From source file:net.peacesoft.nutch.crawl.ReSolrWriter.java
License:Apache License
public void write(NutchDocument doc) throws IOException { final SolrInputDocument inputDoc = new SolrInputDocument(); for (final Entry<String, NutchField> e : doc) { for (final Object val : e.getValue().getValues()) { // normalise the string representation for a Date Object val2 = val; if (val instanceof Date) { val2 = DateUtil.getThreadLocalDateFormat().format(val); }//from www .ja v a 2 s . co m if (e.getKey().equals("content")) { val2 = SolrUtils.stripNonCharCodepoints((String) val); } inputDoc.addField(solrMapping.mapKey(e.getKey()), val2, e.getValue().getWeight()); String sCopy = solrMapping.mapCopyKey(e.getKey()); if (sCopy != e.getKey()) { inputDoc.addField(sCopy, val); } } } postHttp("http://beta2.chodientu.vn/crawler/new", doc); inputDoc.setDocumentBoost(doc.getWeight()); inputDocs.add(inputDoc); if (inputDocs.size() + numDeletes >= commitSize) { try { LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); numDeletes = 0; UpdateRequest req = new UpdateRequest(); req.add(inputDocs); req.setParams(params); req.process(solr); } catch (final SolrServerException e) { throw makeIOException(e); } inputDocs.clear(); } }
From source file:net.peacesoft.nutch.crawl.ReSolrWriter.java
License:Apache License
public void close() throws IOException { try {// w w w . j ava 2 s . c o m if (!inputDocs.isEmpty()) { LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); if (numDeletes > 0) { LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); } UpdateRequest req = new UpdateRequest(); req.add(inputDocs); req.setParams(params); req.process(solr); inputDocs.clear(); } } catch (final SolrServerException e) { throw makeIOException(e); } }
From source file:org.apache.nifi.processors.solr.PutSolrRecord.java
License:Apache License
private void index(boolean isSolrCloud, String collection, Long commitWithin, String contentStreamPath, MultiMapSolrParams requestParams, List<SolrInputDocument> inputDocumentList) throws IOException, SolrServerException, SolrException { UpdateRequest request = new UpdateRequest(contentStreamPath); request.setParams(new ModifiableSolrParams()); // add the extra params, don't use 'set' in case of repeating params Iterator<String> paramNames = requestParams.getParameterNamesIterator(); while (paramNames.hasNext()) { String paramName = paramNames.next(); for (String paramValue : requestParams.getParams(paramName)) { request.getParams().add(paramName, paramValue); }// w w w . ja va 2 s.c om } // specify the collection for SolrCloud if (isSolrCloud) { request.setParam(COLLECTION_PARAM_NAME, collection); } if (commitWithin != null && commitWithin > 0) { request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString()); } // if a username and password were provided then pass them for basic auth if (isBasicAuthEnabled()) { request.setBasicAuthCredentials(getUsername(), getPassword()); } request.add(inputDocumentList); UpdateResponse response = request.process(getSolrClient()); getLogger().debug("Got {} response from Solr", new Object[] { response.getStatus() }); inputDocumentList.clear(); }
From source file:org.apache.nutch.indexer.solr.SolrWriter.java
License:Apache License
public void write(NutchDocument doc) throws IOException { final SolrInputDocument inputDoc = new SolrInputDocument(); for (final Entry<String, NutchField> e : doc) { for (final Object val : e.getValue().getValues()) { // normalise the string representation for a Date Object val2 = val; if (val instanceof Date) { val2 = DateUtil.getThreadLocalDateFormat().format(val); }/* ww w . j av a 2s . c om*/ if (e.getKey().equals("content")) { val2 = SolrUtils.stripNonCharCodepoints((String) val); } inputDoc.addField(solrMapping.mapKey(e.getKey()), val2, e.getValue().getWeight()); String sCopy = solrMapping.mapCopyKey(e.getKey()); if (sCopy != e.getKey()) { inputDoc.addField(sCopy, val); } } } inputDoc.setDocumentBoost(doc.getWeight()); inputDocs.add(inputDoc); if (inputDocs.size() + numDeletes >= commitSize) { try { LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); numDeletes = 0; UpdateRequest req = new UpdateRequest(); req.add(inputDocs); req.setParams(params); req.process(solr); } catch (final SolrServerException e) { throw makeIOException(e); } inputDocs.clear(); } }
From source file:org.apache.nutch.indexer.solr.SolrWriter.java
License:Apache License
public void close() throws IOException { try {//ww w.j a v a 2 s . c o m if (!inputDocs.isEmpty()) { LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); if (numDeletes > 0) { LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); } UpdateRequest req = new UpdateRequest(); req.add(inputDocs); req.setParams(params); req.process(solr); inputDocs.clear(); } // solr.commit(); } catch (final SolrServerException e) { throw makeIOException(e); } }
From source file:org.codelibs.elasticsearch.solr.solr.JavaBinUpdateRequestCodec.java
License:Apache License
/** * Reads a NamedList from the given InputStream, converts it into a * SolrInputDocument and passes it to the given StreamingUpdateHandler * * @param is/*from w w w. ja v a2 s . co m*/ * the InputStream from which to read * @param handler * an instance of StreamingUpdateHandler to which * SolrInputDocuments are streamed one by one * * @return the UpdateRequest * * @throws IOException * in case of an exception while reading from the input stream * or unmarshalling */ public UpdateRequest unmarshal(final InputStream is, final StreamingUpdateHandler handler) throws IOException { final UpdateRequest updateRequest = new UpdateRequest(); List<Object> doclist; // mocksolrplugin: changed to Object List<String> delById; List<String> delByQ; final NamedList[] namedList = new NamedList[1]; final JavaBinCodec codec = new JavaBinCodec() { // NOTE: this only works because this is an anonymous inner class // which will only ever be used on a single stream -- if this class // is ever refactored, this will not work. private boolean seenOuterMostDocIterator = false; @Override public NamedList readNamedList(final DataInputInputStream dis) throws IOException { final int sz = readSize(dis); final NamedList nl = new NamedList(); if (namedList[0] == null) { namedList[0] = nl; } for (int i = 0; i < sz; i++) { final String name = (String) readVal(dis); final Object val = readVal(dis); nl.add(name, val); } return nl; } @Override public List readIterator(final DataInputInputStream fis) throws IOException { // default behavior for reading any regular Iterator in the // stream if (seenOuterMostDocIterator) { return super.readIterator(fis); } // special treatment for first outermost Iterator // (the list of documents) seenOuterMostDocIterator = true; return readOuterMostDocIterator(fis); } private List readOuterMostDocIterator(final DataInputInputStream fis) throws IOException { final NamedList params = (NamedList) namedList[0].getVal(0); updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params))); if (handler == null) { return super.readIterator(fis); } while (true) { final Object o = readVal(fis); if (o == END_OBJ) { break; } SolrInputDocument sdoc = null; if (o instanceof List) { sdoc = JavaBinUpdateRequestCodec.this.listToSolrInputDocument((List<NamedList>) o); } else if (o instanceof NamedList) { final UpdateRequest req = new UpdateRequest(); req.setParams(new ModifiableSolrParams(SolrParams.toSolrParams((NamedList) o))); handler.update(null, req); } else { sdoc = (SolrInputDocument) o; } handler.update(sdoc, updateRequest); } return Collections.EMPTY_LIST; } }; codec.unmarshal(is); // NOTE: if the update request contains only delete commands the params // must be loaded now if (updateRequest.getParams() == null) { final NamedList params = (NamedList) namedList[0].get("params"); if (params != null) { updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params))); } } delById = (List<String>) namedList[0].get("delById"); delByQ = (List<String>) namedList[0].get("delByQ"); doclist = (List) namedList[0].get("docs"); if (doclist != null && !doclist.isEmpty()) { final List<SolrInputDocument> solrInputDocs = new ArrayList<SolrInputDocument>(); for (final Object o : doclist) { if (o instanceof List) { solrInputDocs.add(listToSolrInputDocument((List<NamedList>) o)); } else { solrInputDocs.add((SolrInputDocument) o); } } updateRequest.add(solrInputDocs); } if (delById != null) { for (final String s : delById) { updateRequest.deleteById(s); } } if (delByQ != null) { for (final String s : delByQ) { updateRequest.deleteByQuery(s); } } return updateRequest; }