List of usage examples for org.apache.solr.common.util ContentStream getStream
InputStream getStream() throws IOException;
From source file:com.frank.search.solr.core.schema.SolrJsonRequest.java
License:Apache License
private String quietlyReadContentStreams() { StringBuilder sb = new StringBuilder(); if (contentStream != null) { for (ContentStream stream : this.contentStream) { InputStream ioStream = null; try { ioStream = stream.getStream(); sb.append(StreamUtils.copyToString(ioStream, Charset.forName("UTF-8"))); } catch (IOException e) { } finally { if (ioStream != null) { try { ioStream.close(); } catch (IOException e) { }//from w w w .j a v a 2 s .c o m } } } } return sb.toString(); }
From source file:com.nkang.kxmoment.util.SolrUtils.HttpSolrServer.java
License:Apache License
public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { HttpRequestBase method = null;/*ww w . j a v a2s . c o m*/ InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = DEFAULT_PATH; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = this.parser; } // The parser 'wt=' and 'version=' params are used instead of the original // params ModifiableSolrParams wparams = new ModifiableSolrParams(params); if (parser != null) { wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); } if (invariantParams != null) { wparams.add(invariantParams); } int tries = maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new HttpGet(baseUrl + path + ClientUtils.toQueryString(wparams, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = baseUrl + path; boolean hasNullStreamName = false; if (streams != null) { for (ContentStream cs : streams) { if (cs.getName() == null) { hasNullStreamName = true; break; } } } boolean isMultipart = (this.useMultiPartPost || (streams != null && streams.size() > 1)) && !hasNullStreamName; // only send this list of params as query string params ModifiableSolrParams queryParams = new ModifiableSolrParams(); for (String param : this.queryParams) { String[] value = wparams.getParams(param); if (value != null) { for (String v : value) { queryParams.add(param, v); } wparams.remove(param); } } LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>(); if (streams == null || isMultipart) { HttpPost post = new HttpPost(url + ClientUtils.toQueryString(queryParams, false)); post.setHeader("Content-Charset", "UTF-8"); if (!isMultipart) { post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<FormBodyPart> parts = new LinkedList<FormBodyPart>(); Iterator<String> iter = wparams.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = wparams.getParams(p); if (vals != null) { for (String v : vals) { if (isMultipart) { parts.add(new FormBodyPart(p, new StringBody(v, Charset.forName("UTF-8")))); } else { postParams.add(new BasicNameValuePair(p, v)); } } } } if (isMultipart && streams != null) { for (ContentStream content : streams) { String contentType = content.getContentType(); if (contentType == null) { contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default } String name = content.getName(); if (name == null) { name = ""; } parts.add(new FormBodyPart(name, new InputStreamBody(content.getStream(), contentType, content.getName()))); } } if (parts.size() > 0) { MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT); for (FormBodyPart p : parts) { entity.addPart(p); } post.setEntity(entity); } else { //not using multipart post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8")); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(wparams, false); HttpPost post = new HttpPost(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } else { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if (tries < 1) { throw r; } } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } // client already has this set, is this needed method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects); method.addHeader("User-Agent", AGENT); // add jauth information //String username = WebServiceLoaderUtils.getWebServiceProperty(Constants.SECURITY_USERNAME).toString(); //String password = WebServiceLoaderUtils.getWebServiceProperty(Constants.SECURITY_PASSWORD).toString(); ResourceBundle bundle = ResourceBundle.getBundle("solrconfig"); String username; String password; username = bundle.getString("wsUsername.url"); password = bundle.getString("wsPassword.url"); method.addHeader("username", username); method.addHeader("password", password); InputStream respBody = null; boolean shouldClose = true; boolean success = false; try { // Execute the method. final HttpResponse response = httpClient.execute(method); int httpStatus = response.getStatusLine().getStatusCode(); // Read the contents respBody = response.getEntity().getContent(); Header ctHeader = response.getLastHeader("content-type"); String contentType; if (ctHeader != null) { contentType = ctHeader.getValue(); } else { contentType = ""; } // handle some http level checks before trying to parse the response switch (httpStatus) { case HttpStatus.SC_OK: case HttpStatus.SC_BAD_REQUEST: case HttpStatus.SC_CONFLICT: // 409 break; case HttpStatus.SC_MOVED_PERMANENTLY: case HttpStatus.SC_MOVED_TEMPORARILY: if (!followRedirects) { throw new SolrServerException( "Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ")."); } break; default: if (processor == null) { throw new RemoteSolrException(httpStatus, "Server at " + getBaseURL() + " returned non ok status:" + httpStatus + ", message:" + response.getStatusLine().getReasonPhrase(), null); } } if (processor == null) { // no processor specified, return raw stream NamedList<Object> rsp = new NamedList<Object>(); rsp.add("stream", respBody); // Only case where stream should not be closed shouldClose = false; success = true; return rsp; } String procCt = processor.getContentType(); if (procCt != null) { String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT); String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT); if (!procMimeType.equals(mimeType)) { // unexpected mime type String msg = "Expected mime type " + procMimeType + " but got " + mimeType + "."; Header encodingHeader = response.getEntity().getContentEncoding(); String encoding; if (encodingHeader != null) { encoding = encodingHeader.getValue(); } else { encoding = "UTF-8"; // try UTF-8 } try { msg = msg + " " + IOUtils.toString(respBody, encoding); } catch (IOException e) { throw new RemoteSolrException(httpStatus, "Could not parse response with encoding " + encoding, e); } RemoteSolrException e = new RemoteSolrException(httpStatus, msg, null); throw e; } } // if(true) { // ByteArrayOutputStream copy = new ByteArrayOutputStream(); // IOUtils.copy(respBody, copy); // String val = new String(copy.toByteArray()); // System.out.println(">RESPONSE>"+val+"<"+val.length()); // respBody = new ByteArrayInputStream(copy.toByteArray()); // } NamedList<Object> rsp = null; String charset = EntityUtils.getContentCharSet(response.getEntity()); try { rsp = processor.processResponse(respBody, charset); } catch (Exception e) { throw new RemoteSolrException(httpStatus, e.getMessage(), e); } if (httpStatus != HttpStatus.SC_OK) { String reason = null; try { NamedList<?> err = (NamedList<?>) rsp.get("error"); if (err != null) { reason = (String) err.get("msg"); // TODO? get the trace? } } catch (Exception ex) { } if (reason == null) { StringBuilder msg = new StringBuilder(); msg.append(response.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append("request: " + method.getURI()); reason = java.net.URLDecoder.decode(msg.toString(), UTF_8); } throw new RemoteSolrException(httpStatus, reason, null); } success = true; return rsp; } catch (ConnectException e) { throw new SolrServerException("Server refused connection at: " + getBaseURL(), e); } catch (SocketTimeoutException e) { throw new SolrServerException("Timeout occured while waiting response from server at: " + getBaseURL(), e); } catch (IOException e) { throw new SolrServerException("IOException occured when talking to server at: " + getBaseURL(), e); } finally { if (respBody != null && shouldClose) { try { respBody.close(); } catch (IOException e) { //log.error("", e); } finally { if (!success) { method.abort(); } } } } }
From source file:com.su.search.client.solrj.PaHttpSolrServer.java
License:Apache License
public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { HttpRequestBase method = null;//from w w w .j av a 2s. c o m InputStream is = null; SolrParams params = request.getParams(); // modified by wangqiang406 2012-07-27 // ?? if (null != password) { ModifiableSolrParams wparams = new ModifiableSolrParams(params); wparams.set(SimpleIndexClient.KEY_PA_AUTH, password); params = wparams; } Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = DEFAULT_PATH; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = this.parser; } // The parser 'wt=' and 'version=' params are used instead of the original // params ModifiableSolrParams wparams = new ModifiableSolrParams(params); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (invariantParams != null) { wparams.add(invariantParams); } params = wparams; int tries = maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new HttpGet(baseUrl + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = baseUrl + path; boolean isMultipart = (streams != null && streams.size() > 1); LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>(); if (streams == null || isMultipart) { HttpPost post = new HttpPost(url); post.setHeader("Content-Charset", "UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<FormBodyPart> parts = new LinkedList<FormBodyPart>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new FormBodyPart(p, new StringBody(v, Charset.forName("UTF-8")))); } else { postParams.add(new BasicNameValuePair(p, v)); } } } } if (isMultipart) { for (ContentStream content : streams) { parts.add(new FormBodyPart(content.getName(), new InputStreamBody(content.getStream(), content.getName()))); } } if (parts.size() > 0) { MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT); for (FormBodyPart p : parts) { entity.addPart(p); } post.setEntity(entity); } else { //not using multipart HttpEntity e; post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8")); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); HttpPost post = new HttpPost(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } else { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if (tries < 1) { throw r; } } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } // TODO: move to a interceptor? method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects); method.addHeader("User-Agent", AGENT); InputStream respBody = null; try { // Execute the method. final HttpResponse response = httpClient.execute(method); int httpStatus = response.getStatusLine().getStatusCode(); // Read the contents String charset = EntityUtils.getContentCharSet(response.getEntity()); respBody = response.getEntity().getContent(); // handle some http level checks before trying to parse the response switch (httpStatus) { case HttpStatus.SC_OK: break; case HttpStatus.SC_MOVED_PERMANENTLY: case HttpStatus.SC_MOVED_TEMPORARILY: if (!followRedirects) { throw new SolrServerException( "Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ")."); } break; case HttpStatus.SC_NOT_FOUND: throw new SolrServerException("Server at " + getBaseURL() + " was not found (404)."); default: throw new SolrServerException("Server at " + getBaseURL() + " returned non ok status:" + httpStatus + ", message:" + response.getStatusLine().getReasonPhrase()); } NamedList<Object> rsp = processor.processResponse(respBody, charset); if (httpStatus != HttpStatus.SC_OK) { String reason = null; try { NamedList err = (NamedList) rsp.get("error"); if (err != null) { reason = (String) err.get("msg"); // TODO? get the trace? } } catch (Exception ex) { } if (reason == null) { StringBuilder msg = new StringBuilder(); msg.append(response.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append("request: " + method.getURI()); reason = java.net.URLDecoder.decode(msg.toString(), UTF_8); } throw new SolrException(SolrException.ErrorCode.getErrorCode(httpStatus), reason); } return rsp; } catch (ConnectException e) { throw new SolrServerException("Server refused connection at: " + getBaseURL(), e); } catch (SocketTimeoutException e) { throw new SolrServerException("Timeout occured while waiting response from server at: " + getBaseURL(), e); } catch (IOException e) { throw new SolrServerException("IOException occured when talking to server at: " + getBaseURL(), e); } finally { if (respBody != null) { try { respBody.close(); } catch (Throwable t) { } // ignore } } }
From source file:lux.solr.XQueryComponent.java
License:Mozilla Public License
private void handleContentStreams(LinkedTreeBuilder builder, SolrQueryRequest req, ArrayList<XdmItem> result, Evaluator evaluator) throws XPathException { // parts//from w ww . ja v a2 s . c om int i = 0; for (ContentStream stream : req.getContentStreams()) { String contentType = stream.getContentType(); //String name = stream.getName(); byte[] partBytes = null; try { partBytes = IOUtils.toByteArray(stream.getStream(), stream.getSize()); } catch (IOException e) { throw new LuxException(e); } String charset = ContentStreamBase.getCharsetFromContentType(contentType); if (charset == null) { charset = "utf-8"; } if (!isText(contentType)) { logger.warn("Binary values not supported; treating " + contentType + " as xml, or text"); } XdmItem part = null; if (isXML(contentType) || !isText(contentType)) { try { part = evaluator.build(new ByteArrayInputStream(partBytes), "#part" + i); } catch (LuxException e) { // failed to parse logger.warn("Caught an exception while parsing XML: " + e.getMessage() + ", treating it as plain text"); contentType = "text/plain; charset=" + charset; } } if (part == null) { String text; try { text = new String(partBytes, charset); } catch (UnsupportedEncodingException e1) { throw new LuxException(e1); } if (isHTML(contentType)) { HtmlParser parser = new HtmlParser(); //Parser parser = new Parser(); SAXSource source = new SAXSource(parser, new InputSource(new StringReader(text))); try { part = evaluator.getDocBuilder().build(source); } catch (SaxonApiException e) { e.printStackTrace(); logger.warn("failed to parse HTML; treating as plain text: " + e.getMessage()); } } if (part == null) { TextFragmentValue node = new TextFragmentValue(text, "#part" + i); node.setConfiguration(builder.getConfiguration()); part = new XdmNode(node); } } result.add(part); builder.startElement(fQNameFor("http", EXPATH_HTTP_NS, "body"), BuiltInAtomicType.UNTYPED_ATOMIC, 0, 0); addAttribute(builder, "position", "1"); addAttribute(builder, "content-type", contentType); builder.startContent(); builder.endElement(); } }
From source file:org.alfresco.solr.query.AbstractQParser.java
License:Open Source License
protected Pair<SearchParameters, Boolean> getSearchParameters() { SearchParameters searchParameters = new SearchParameters(); Boolean isFilter = Boolean.FALSE; Iterable<ContentStream> streams = req.getContentStreams(); JSONObject json = (JSONObject) req.getContext().get(ALFRESCO_JSON); if (json == null) { if (streams != null) { try { Reader reader = null; for (ContentStream stream : streams) { reader = new BufferedReader(new InputStreamReader(stream.getStream(), "UTF-8")); }//w ww .j av a2 s. co m // TODO - replace with streaming-based solution e.g. SimpleJSON ContentHandler if (reader != null) { json = new JSONObject(new JSONTokener(reader)); req.getContext().put(ALFRESCO_JSON, json); } } catch (JSONException e) { // This is expected when there is no json element to the request } catch (IOException e) { throw new AlfrescoRuntimeException("IO Error parsing query parameters", e); } } } if (json != null) { try { if (getString() != null) { if (getString().equals(AUTHORITY_FILTER_FROM_JSON)) { isFilter = Boolean.TRUE; ArrayList<String> tenantList = new ArrayList<String>(1); JSONArray tenants = json.getJSONArray("tenants"); for (int i = 0; i < tenants.length(); i++) { String tenantString = tenants.getString(i); tenantList.add(tenantString); } ArrayList<String> authorityList = new ArrayList<String>(1); JSONArray authorities = json.getJSONArray("authorities"); for (int i = 0; i < authorities.length(); i++) { String authorityString = authorities.getString(i); authorityList.add(authorityString); } char separator = getSeparator(authorityList); StringBuilder authQuery = new StringBuilder(); StringBuilder denyQuery = new StringBuilder(); for (String tenant : tenantList) { for (String authority : authorityList) { if (separator == 0) { if (authQuery.length() > 0) { authQuery.append(" "); denyQuery.append(" "); } switch (AuthorityType.getAuthorityType(authority)) { case USER: authQuery.append("|AUTHORITY:\"").append(authority).append("\""); denyQuery.append("|DENIED:\"").append(authority).append("\""); break; case GROUP: case EVERYONE: case GUEST: if (tenant.length() == 0) { // Default tenant matches 4.0 authQuery.append("|AUTHORITY:\"").append(authority).append("\""); denyQuery.append("|DENIED:\"").append(authority).append("\""); } else { authQuery.append("|AUTHORITY:\"").append(authority).append("@") .append(tenant).append("\""); denyQuery.append("|DENIED:\"").append(authority).append("@") .append(tenant).append("\""); } break; default: authQuery.append("|AUTHORITY:\"").append(authority).append("\""); denyQuery.append("|DENIED:\"").append(authority).append("\""); break; } } else { if (authQuery.length() == 0) { authset = true; authQuery.append("|AUTHSET:\""); denyQuery.append("|DENYSET:\""); } switch (AuthorityType.getAuthorityType(authority)) { case USER: authQuery.append(separator).append(authority); denyQuery.append(separator).append(authority); break; case GROUP: case EVERYONE: case GUEST: if (tenant.length() == 0) { // Default tenant matches 4.0 authQuery.append(separator).append(authority); denyQuery.append(separator).append(authority); } else { authQuery.append(separator).append(authority).append("@") .append(tenant); denyQuery.append(separator).append(authority).append("@") .append(tenant); } break; default: authQuery.append(separator).append(authority); denyQuery.append(separator).append(authority); break; } } } } if (separator != 0) { authQuery.append("\""); denyQuery.append("\""); } if (authQuery.length() > 0) { // Default to true for safety reasons. final boolean anyDenyDenies = json.optBoolean("anyDenyDenies", true); if (anyDenyDenies) { authQuery.insert(0, "(").append(") AND NOT (").append(denyQuery).append(")"); // Record that the clause has been added. // We only ever set this to true for solr4+ req.getContext().put("processedDenies", Boolean.TRUE); } searchParameters.setQuery(authQuery.toString()); } } else if (getString().equals(TENANT_FILTER_FROM_JSON)) { isFilter = Boolean.TRUE; ArrayList<String> tenantList = new ArrayList<String>(1); JSONArray tenants = json.getJSONArray("tenants"); for (int i = 0; i < tenants.length(); i++) { String tenantString = tenants.getString(i); tenantList.add(tenantString); } StringBuilder tenantQuery = new StringBuilder(); for (String tenant : tenantList) { if (tenantQuery.length() > 0) { tenantQuery.append(" "); } if (tenant.length() > 0) { tenantQuery.append("|TENANT:\"").append(tenant).append("\""); } else { // TODO: Need to check for the default tenant or no tenant (4.0) or we force a reindex // requirement later ... // Better to add default tenant to the 4.0 index tenantQuery.append("|TENANT:\"").append("_DEFAULT_").append("\""); // tenantQuery.append(" |(+ISNODE:T -TENANT:*)"); } } searchParameters.setQuery(tenantQuery.toString()); } else if (getString().equals(RERANK_QUERY_FROM_CONTEXT)) { String searchTerm = getParam("spellcheck.q"); searchParameters.setQuery(searchTerm); } } else { String query = json.getString("query"); if (query != null) { searchParameters.setQuery(query); } } JSONArray locales = json.getJSONArray("locales"); for (int i = 0; i < locales.length(); i++) { String localeString = locales.getString(i); Locale locale = DefaultTypeConverter.INSTANCE.convert(Locale.class, localeString); searchParameters.addLocale(locale); } JSONArray templates = json.getJSONArray("templates"); for (int i = 0; i < templates.length(); i++) { JSONObject template = templates.getJSONObject(i); String name = template.getString("name"); String queryTemplate = template.getString("template"); searchParameters.addQueryTemplate(name, queryTemplate); } JSONArray allAttributes = json.getJSONArray("allAttributes"); for (int i = 0; i < allAttributes.length(); i++) { String allAttribute = allAttributes.getString(i); searchParameters.addAllAttribute(allAttribute); } searchParameters.setDefaultFTSOperator(Operator.valueOf(json.getString("defaultFTSOperator"))); searchParameters .setDefaultFTSFieldConnective(Operator.valueOf(json.getString("defaultFTSFieldOperator"))); if (json.has("mlAnalaysisMode")) { searchParameters.setMlAnalaysisMode(MLAnalysisMode.valueOf(json.getString("mlAnalaysisMode"))); } searchParameters.setNamespace(json.getString("defaultNamespace")); JSONArray textAttributes = json.getJSONArray("textAttributes"); for (int i = 0; i < textAttributes.length(); i++) { String textAttribute = textAttributes.getString(i); searchParameters.addTextAttribute(textAttribute); } searchParameters.setQueryConsistency(QueryConsistency.valueOf(json.getString("queryConsistency"))); } catch (JSONException e) { // This is expected when there is no json element to the request } } if (json != null) { if (log.isDebugEnabled()) { log.debug(json.toString()); } } if (searchParameters.getQuery() == null) { searchParameters.setQuery(getString()); } if (searchParameters.getLocales().size() == 0) { searchParameters.addLocale(I18NUtil.getLocale()); } String defaultField = getParam(CommonParams.DF); if (defaultField != null) { searchParameters.setDefaultFieldName(defaultField); } if (autoDetectQueryLocale) { String searchTerm = getParam("spellcheck.q"); if (searchTerm != null) { searchParameters.setSearchTerm(searchTerm); List<DetectedLanguage> detetcted = detectLanguage(searchTerm); if ((detetcted != null) && (detetcted.size() > 0)) { Locale detectedLocale = Locale.forLanguageTag(detetcted.get(0).getLangCode()); if (localeIsNotIncluded(searchParameters, detectedLocale)) { searchParameters.addLocale(Locale.forLanguageTag(detectedLocale.getLanguage())); } } } } if (fixedQueryLocales.size() > 0) { for (String locale : fixedQueryLocales) { searchParameters.addLocale(Locale.forLanguageTag(locale)); } } // searchParameters.setMlAnalaysisMode(getMLAnalysisMode()); searchParameters.setNamespace(NamespaceService.CONTENT_MODEL_1_0_URI); return new Pair<SearchParameters, Boolean>(searchParameters, isFilter); }
From source file:org.apache.manifoldcf.agents.output.solr.ModifiedHttpSolrServer.java
License:Apache License
@Override public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { HttpRequestBase method = null;/*ww w. j a v a 2 s .c om*/ InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = DEFAULT_PATH; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = this.parser; } // The parser 'wt=' and 'version=' params are used instead of the original // params ModifiableSolrParams wparams = new ModifiableSolrParams(params); if (parser != null) { wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); } if (invariantParams != null) { wparams.add(invariantParams); } params = wparams; int tries = maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new HttpGet(baseUrl + path + toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = baseUrl + path; boolean hasNullStreamName = false; if (streams != null) { for (ContentStream cs : streams) { if (cs.getName() == null) { hasNullStreamName = true; break; } } } boolean isMultipart = (this.useMultiPartPost || (streams != null && streams.size() > 1)) && !hasNullStreamName; LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>(); if (streams == null || isMultipart) { HttpPost post = new HttpPost(url); post.setHeader("Content-Charset", "UTF-8"); if (!isMultipart) { post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<FormBodyPart> parts = new LinkedList<FormBodyPart>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (isMultipart) { parts.add( new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8))); } else { postParams.add(new BasicNameValuePair(p, v)); } } } } if (isMultipart && streams != null) { for (ContentStream content : streams) { String contentType = content.getContentType(); if (contentType == null) { contentType = "application/octet-stream"; // default } String contentName = content.getName(); parts.add(new FormBodyPart(contentName, new InputStreamBody(content.getStream(), contentType, content.getName()))); } } if (parts.size() > 0) { ModifiedMultipartEntity entity = new ModifiedMultipartEntity( HttpMultipartMode.STRICT, null, StandardCharsets.UTF_8); for (FormBodyPart p : parts) { entity.addPart(p); } post.setEntity(entity); } else { //not using multipart post.setEntity(new UrlEncodedFormEntity(postParams, StandardCharsets.UTF_8)); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = toQueryString(params, false); HttpPost post = new HttpPost(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } else { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if (tries < 1) { throw r; } } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } // XXX client already has this set, is this needed? //method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, // followRedirects); method.addHeader("User-Agent", AGENT); InputStream respBody = null; boolean shouldClose = true; try { // Execute the method. final HttpResponse response = httpClient.execute(method); int httpStatus = response.getStatusLine().getStatusCode(); // Read the contents respBody = response.getEntity().getContent(); // handle some http level checks before trying to parse the response switch (httpStatus) { case HttpStatus.SC_OK: case HttpStatus.SC_BAD_REQUEST: case HttpStatus.SC_CONFLICT: // 409 break; case HttpStatus.SC_MOVED_PERMANENTLY: case HttpStatus.SC_MOVED_TEMPORARILY: if (!followRedirects) { throw new SolrServerException( "Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ")."); } break; default: throw new SolrException(SolrException.ErrorCode.getErrorCode(httpStatus), "Server at " + getBaseURL() + " returned non ok status:" + httpStatus + ", message:" + response.getStatusLine().getReasonPhrase()); } if (processor == null) { // no processor specified, return raw stream NamedList<Object> rsp = new NamedList<Object>(); rsp.add("stream", respBody); // Only case where stream should not be closed shouldClose = false; return rsp; } Charset charsetObject = ContentType.getOrDefault(response.getEntity()).getCharset(); String charset; if (charsetObject != null) charset = charsetObject.name(); else charset = "utf-8"; NamedList<Object> rsp = processor.processResponse(respBody, charset); if (httpStatus != HttpStatus.SC_OK) { String reason = null; try { NamedList err = (NamedList) rsp.get("error"); if (err != null) { reason = (String) err.get("msg"); // TODO? get the trace? } } catch (Exception ex) { } if (reason == null) { StringBuilder msg = new StringBuilder(); msg.append(response.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append("request: " + method.getURI()); reason = URLDecoder.decode(msg.toString()); } throw new SolrException(SolrException.ErrorCode.getErrorCode(httpStatus), reason); } return rsp; } catch (ConnectException e) { throw new SolrServerException("Server refused connection at: " + getBaseURL(), e); } catch (SocketTimeoutException e) { throw new SolrServerException("Timeout occured while waiting response from server at: " + getBaseURL(), e); } catch (IOException e) { throw new SolrServerException("IOException occured when talking to server at: " + getBaseURL(), e); } finally { if (respBody != null && shouldClose) { try { respBody.close(); } catch (Throwable t) { } // ignore } } }
From source file:org.sleuthkit.autopsy.keywordsearch.Ingester.java
License:Open Source License
/** * Indexing method that bypasses Tika, assumes pure text * It reads and converts the entire content stream to string, assuming UTF8 * since we can't use streaming approach for Solr /update handler. * This should be safe, since all content is now in max 1MB chunks. * /*from ww w . jav a2 s . c o m*/ * TODO see if can use a byte or string streaming way to add content to /update handler * e.g. with XMLUpdateRequestHandler (deprecated in SOlr 4.0.0), see if possible * to stream with UpdateRequestHandler * * @param cs * @param fields * @param size * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException */ void ingest(ContentStream cs, Map<String, String> fields, final long size) throws IngesterException { if (fields.get(Server.Schema.IMAGE_ID.toString()) == null) { //skip the file, image id unknown String msg = NbBundle.getMessage(this.getClass(), "Ingester.ingest.exception.unknownImgId.msg", cs.getName()); logger.log(Level.SEVERE, msg); throw new IngesterException(msg); } final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE]; SolrInputDocument updateDoc = new SolrInputDocument(); for (String key : fields.keySet()) { updateDoc.addField(key, fields.get(key)); } //using size here, but we are no longer ingesting entire files //size is normally a chunk size, up to 1MB if (size > 0) { InputStream is = null; int read = 0; try { is = cs.getStream(); read = is.read(docChunkContentBuf); } catch (IOException ex) { throw new IngesterException(NbBundle.getMessage(this.getClass(), "Ingester.ingest.exception.cantReadStream.msg", cs.getName())); } finally { try { is.close(); } catch (IOException ex) { logger.log(Level.WARNING, "Could not close input stream after reading content, " + cs.getName(), ex); //NON-NLS } } if (read != 0) { String s = ""; try { s = new String(docChunkContentBuf, 0, read, docContentEncoding); } catch (UnsupportedEncodingException ex) { Exceptions.printStackTrace(ex); } updateDoc.addField(Server.Schema.CONTENT.toString(), s); } else { updateDoc.addField(Server.Schema.CONTENT.toString(), ""); } } else { //no content, such as case when 0th chunk indexed updateDoc.addField(Server.Schema.CONTENT.toString(), ""); } try { //TODO consider timeout thread, or vary socket timeout based on size of indexed content solrServer.addDocument(updateDoc); uncommitedIngests = true; } catch (KeywordSearchModuleException ex) { throw new IngesterException( NbBundle.getMessage(this.getClass(), "Ingester.ingest.exception.err.msg", cs.getName()), ex); } }
From source file:org.vootoo.client.netty.NettyUtil.java
License:Apache License
public static ByteString readFrom(ContentStream stream) throws IOException { InputStream in = stream.getStream(); return ByteString.readFrom(in); }
From source file:org.vootoo.client.netty.util.ProtobufUtil.java
License:Apache License
public static List<SolrProtocol.ContentStream> toProtobufContentStreams( Collection<ContentStream> contentStreams) { return Lists.transform(Lists.newArrayList(contentStreams), new Function<ContentStream, SolrProtocol.ContentStream>() { @Override/*from w w w. j a v a 2 s.co m*/ public SolrProtocol.ContentStream apply(ContentStream input) { SolrProtocol.ContentStream.Builder stream = SolrProtocol.ContentStream.newBuilder(); if (input.getName() != null) { stream.setName(input.getName()); } String contentType = input.getContentType(); if (contentType == null) { // default javabin contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; } stream.setContentType(contentType); stream.setSourceInfo(input.getSourceInfo()); stream.setSize(input.getSize()); try { //TODO zero copy byte stream.setStream(ByteString.readFrom(input.getStream())); } catch (IOException e) { //TODO dealing input.getStream() exception throw new RuntimeException("ContentStream.getStream() exception", e); } return stream.build(); } }); }
From source file:sk.datalan.solr.impl.HttpSolrServer.java
License:Apache License
protected HttpRequestBase createMethod(final SolrRequest request) throws IOException, SolrServerException { HttpRequestBase method = null;/*from w ww. j a va 2s. c o m*/ InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = DEFAULT_PATH; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = this.parser; } // The parser 'wt=' and 'version=' params are used instead of the original // params ModifiableSolrParams wparams = new ModifiableSolrParams(params); if (parser != null) { wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); } if (invariantParams != null) { wparams.add(invariantParams); } int tries = maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new HttpGet(baseUrl + path + ClientUtils.toQueryString(wparams, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = baseUrl + path; boolean hasNullStreamName = false; if (streams != null) { for (ContentStream cs : streams) { if (cs.getName() == null) { hasNullStreamName = true; break; } } } boolean isMultipart = (this.useMultiPartPost || (streams != null && streams.size() > 1)) && !hasNullStreamName; // only send this list of params as query string params ModifiableSolrParams queryParams = new ModifiableSolrParams(); for (String param : this.queryParams) { String[] value = wparams.getParams(param); if (value != null) { for (String v : value) { queryParams.add(param, v); } wparams.remove(param); } } LinkedList<NameValuePair> postParams = new LinkedList<>(); if (streams == null || isMultipart) { HttpPost post = new HttpPost(url + ClientUtils.toQueryString(queryParams, false)); post.setHeader("Content-Charset", "UTF-8"); if (!isMultipart) { post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<FormBodyPart> parts = new LinkedList<>(); Iterator<String> iter = wparams.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = wparams.getParams(p); if (vals != null) { for (String v : vals) { if (isMultipart) { parts.add( new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8))); } else { postParams.add(new BasicNameValuePair(p, v)); } } } } if (isMultipart && streams != null) { for (ContentStream content : streams) { String contentType = content.getContentType(); if (contentType == null) { contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default } String name = content.getName(); if (name == null) { name = ""; } parts.add(new FormBodyPart(name, new InputStreamBody(content.getStream(), contentType, content.getName()))); } } if (parts.size() > 0) { MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT); for (FormBodyPart p : parts) { entity.addPart(p); } post.setEntity(entity); } else { //not using multipart post.setEntity(new UrlEncodedFormEntity(postParams, StandardCharsets.UTF_8)); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(wparams, false); HttpPost post = new HttpPost(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } else { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if (tries < 1) { throw r; } } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } return method; }