List of usage examples for org.apache.solr.client.solrj.request ContentStreamUpdateRequest getParams
@Override
public ModifiableSolrParams getParams()
From source file:org.apache.nifi.processors.solr.PutSolrContentStream.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get();// w ww . j a va 2s . co m if (flowFile == null) { return; } final AtomicReference<Exception> error = new AtomicReference<>(null); final AtomicReference<Exception> connectionError = new AtomicReference<>(null); final boolean isSolrCloud = SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue()); final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions(flowFile).getValue(); final Long commitWithin = context.getProperty(COMMIT_WITHIN).evaluateAttributeExpressions(flowFile) .asLong(); final String contentStreamPath = context.getProperty(CONTENT_STREAM_PATH) .evaluateAttributeExpressions(flowFile).getValue(); final MultiMapSolrParams requestParams = new MultiMapSolrParams(getRequestParams(context, flowFile)); StopWatch timer = new StopWatch(true); session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(contentStreamPath); request.setParams(new ModifiableSolrParams()); // add the extra params, don't use 'set' in case of repeating params Iterator<String> paramNames = requestParams.getParameterNamesIterator(); while (paramNames.hasNext()) { String paramName = paramNames.next(); for (String paramValue : requestParams.getParams(paramName)) { request.getParams().add(paramName, paramValue); } } // specify the collection for SolrCloud if (isSolrCloud) { request.setParam(COLLECTION_PARAM_NAME, collection); } if (commitWithin != null && commitWithin > 0) { request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString()); } // if a username and password were provided then pass them for basic auth if (isBasicAuthEnabled()) { request.setBasicAuthCredentials(getUsername(), getPassword()); } try (final BufferedInputStream bufferedIn = new BufferedInputStream(in)) { // add the FlowFile's content on the UpdateRequest request.addContentStream(new ContentStreamBase() { @Override public InputStream getStream() throws IOException { return bufferedIn; } @Override public String getContentType() { return context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions().getValue(); } }); UpdateResponse response = request.process(getSolrClient()); getLogger().debug("Got {} response from Solr", new Object[] { response.getStatus() }); } catch (SolrException e) { error.set(e); } catch (SolrServerException e) { if (causedByIOException(e)) { connectionError.set(e); } else { error.set(e); } } catch (IOException e) { connectionError.set(e); } } }); timer.stop(); if (error.get() != null) { getLogger().error("Failed to send {} to Solr due to {}; routing to failure", new Object[] { flowFile, error.get() }); session.transfer(flowFile, REL_FAILURE); } else if (connectionError.get() != null) { getLogger().error("Failed to send {} to Solr due to {}; routing to connection_failure", new Object[] { flowFile, connectionError.get() }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_CONNECTION_FAILURE); } else { StringBuilder transitUri = new StringBuilder("solr://"); transitUri.append(getSolrLocation()); if (isSolrCloud) { transitUri.append(":").append(collection); } final long duration = timer.getDuration(TimeUnit.MILLISECONDS); session.getProvenanceReporter().send(flowFile, transitUri.toString(), duration, true); getLogger().info("Successfully sent {} to Solr in {} millis", new Object[] { flowFile, duration }); session.transfer(flowFile, REL_SUCCESS); } }
From source file:org.craftercms.search.service.impl.SolrDocumentBuilder.java
License:Open Source License
/** * Build the Solr document for partial update of the search engine's index data of a structured document. * * @param request Content Stream update request for document update * @param additionalFields Fields to add to solr document * @return the Solr document/*from w w w . j a v a 2s.c om*/ * */ public ContentStreamUpdateRequest buildPartialUpdateDocument(ContentStreamUpdateRequest request, Map<String, String> additionalFields) { for (Map.Entry<String, String> additionalField : additionalFields.entrySet()) { String fieldName = additionalField.getKey(); String fieldValue = additionalField.getValue(); // If fieldName ends with HTML prefix, strip all HTML markup from the field value. if (fieldName.endsWith(htmlFieldSuffix)) { if (logger.isDebugEnabled()) { logger.debug("Stripping HTML from field '" + fieldName + "'"); } fieldValue = stripHtml(fieldName, fieldValue); } // If fieldName ends with datetime prefix, convert the field value to an ISO datetime string. if (fieldName.endsWith(dateTimeFieldSuffix)) { if (logger.isDebugEnabled()) { logger.debug("Converting '" + fieldValue + "' to ISO datetime"); } fieldValue = convertToISODateTimeString(fieldValue); } if (logger.isDebugEnabled()) { logger.debug("Adding field '" + fieldName + "' to the Solr doc"); } if (fieldName.endsWith(htmlFieldSuffix) || fieldName.endsWith(dateTimeFieldSuffix)) { request.setParam(ExtractingParams.LITERALS_PREFIX + fieldName, fieldValue); } else { String[] fieldValues = fieldValue.split(multivalueSeparator); if (fieldValues.length > 1) { ModifiableSolrParams params = request.getParams(); params.add(ExtractingParams.LITERALS_PREFIX + fieldName, fieldValues); } else { request.setParam(ExtractingParams.LITERALS_PREFIX + fieldName, fieldValue); } } } return request; }
From source file:org.gss_project.gss.server.ejb.ExternalAPIBean.java
License:Open Source License
private void postFileToSolr(CommonsHttpSolrServer solr, Long id) throws ObjectNotFoundException { try {/*from ww w . j a v a 2s .co m*/ FileHeader file = dao.getFileForIndexing(id); FileBody body = file.getCurrentBody(); String mime = body.getMimeType(); boolean multipart = true; if (!mime.equals("application/pdf") && !mime.equals("text/plain") && !mime.equals("text/html") && !mime.endsWith("msword") && !mime.endsWith("ms-excel") && !mime.endsWith("powerpoint") || (body.getFileSize() > getConfiguration().getLong("solrDocumentUploadLimitInKB") * 1024)) multipart = false; if (!multipart) sendMetaDataOnly(solr, file); else { ContentStreamUpdateRequest solrRequest = new ContentStreamUpdateRequest( getConfiguration().getString("solr.rich.update.path")); solrRequest.setParam("literal.id", file.getId().toString()); solrRequest.setParam("literal.name", file.getName()); for (FileTag t : file.getFileTags()) { solrRequest.getParams().add("literal.tag", t.getTag()); } for (Permission p : file.getPermissions()) { if (p.getRead()) { if (p.getUser() != null) solrRequest.getParams().add("literal.ureaders", p.getUser().getId().toString()); else if (p.getGroup() != null) solrRequest.getParams().add("literal.greaders", p.getGroup().getId().toString()); } } solrRequest.setParam("literal.owner", file.getOwner().getId().toString()); solrRequest.setParam("literal.public", String.valueOf(file.isReadForAll())); File fsFile = new File(body.getStoredFilePath()); solrRequest.addFile(fsFile); try { solr.request(solrRequest); } catch (SolrException e) { logger.warn("File " + id + " failed with SolrException: " + e.getLocalizedMessage() + ". Retrying without the file"); //Let 's try without the file sendMetaDataOnly(solr, file); } catch (NullPointerException e) { logger.warn("File " + id + " failed with NullPointerException: " + e.getLocalizedMessage() + ". Retrying without the file"); //Let 's try without the file sendMetaDataOnly(solr, file); } catch (SolrServerException e) { logger.warn("File " + id + " failed with SolrServerException: " + e.getLocalizedMessage() + ". Retrying without the file"); //Let 's try without the file sendMetaDataOnly(solr, file); } } } catch (MalformedURLException e) { throw new EJBException(e); } catch (SolrServerException e) { throw new EJBException(e); } catch (IOException e) { throw new EJBException(e); } }