List of usage examples for org.apache.commons.httpclient HttpMethod setFollowRedirects
public abstract void setFollowRedirects(boolean paramBoolean);
From source file:org.apache.solr.client.solrj.impl.CommonsHttpSolrServer.java
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { HttpMethod method = null; InputStream is = null;/*from w w w. j av a 2 s . c o m*/ SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = "/select"; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = _parser; } // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (params == null) { params = wparams; } else { params = new DefaultSolrParams(wparams, params); } if (_invariantParams != null) { params = new DefaultSolrParams(_invariantParams, params); } int tries = _maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = _baseURL + path; boolean isMultipart = (streams != null && streams.size() > 1); if (streams == null || isMultipart) { PostMethod post = new PostMethod(url); post.getParams().setContentCharset("UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<Part> parts = new LinkedList<Part>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new StringPart(p, v, "UTF-8")); } else { post.addParameter(p, v); } } } } if (isMultipart) { int i = 0; for (ContentStream content : streams) { final ContentStream c = content; String charSet = null; PartSource source = new PartSource() { public long getLength() { return c.getSize(); } public String getFileName() { return c.getName(); } public InputStream createInputStream() throws IOException { return c.getStream(); } }; parts.add(new FilePart(c.getName(), source, c.getContentType(), charSet)); } } if (parts.size() > 0) { post.setRequestEntity(new MultipartRequestEntity( parts.toArray(new Part[parts.size()]), post.getParams())); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); PostMethod post = new PostMethod(url + pstr); // post.setRequestHeader("connection", "close"); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setRequestEntity(new RequestEntity() { public long getContentLength() { return -1; } public String getContentType() { return contentStream[0].getContentType(); } public boolean isRepeatable() { return false; } public void writeRequest(OutputStream outputStream) throws IOException { ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream); } }); } else { is = contentStream[0].getStream(); post.setRequestEntity( new InputStreamRequestEntity(is, contentStream[0].getContentType())); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { // This is generally safe to retry on method.releaseConnection(); method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if ((tries < 1)) { throw r; } //log.warn( "Caught: " + r + ". Retrying..." ); } } } catch (IOException ex) { log.error("####request####", ex); throw new SolrServerException("error reading streams", ex); } method.setFollowRedirects(_followRedirects); method.addRequestHeader("User-Agent", AGENT); if (_allowCompression) { method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate")); } // method.setRequestHeader("connection", "close"); try { // Execute the method. //System.out.println( "EXECUTE:"+method.getURI() ); int statusCode = _httpClient.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { StringBuilder msg = new StringBuilder(); msg.append(method.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append(method.getStatusText()); msg.append("\n\n"); msg.append("request: " + method.getURI()); throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8")); } // Read the contents String charset = "UTF-8"; if (method instanceof HttpMethodBase) { charset = ((HttpMethodBase) method).getResponseCharSet(); } InputStream respBody = method.getResponseBodyAsStream(); // Jakarta Commons HTTPClient doesn't handle any // compression natively. Handle gzip or deflate // here if applicable. if (_allowCompression) { Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.contains("gzip")) { //log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentEncoding.contains("deflate")) { //log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } else { Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { String contentType = contentTypeHeader.getValue(); if (contentType != null) { if (contentType.startsWith("application/x-gzip-compressed")) { //log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentType.startsWith("application/x-deflate")) { //log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } } } } return processor.processResponse(respBody, charset); } catch (HttpException e) { throw new SolrServerException(e); } catch (IOException e) { throw new SolrServerException(e); } finally { method.releaseConnection(); if (is != null) { is.close(); } } }
From source file:org.archive.crawler.fetcher.FetchHTTP.java
/** * Configure the HttpMethod setting options and headers. * * @param curi CrawlURI from which we pull configuration. * @param method The Method to configure. * @return HostConfiguration copy customized for this CrawlURI *///from www. j a v a 2 s . co m protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); // // set soTimeout // method.getParams().setSoTimeout( // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) // .intValue()); // Set cookie policy. method.getParams() .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue()) ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); // Use only HTTP/1.0 (to avoid receiving chunked responses) method.getParams().setVersion(HttpVersion.HTTP_1_0); CrawlOrder order = getSettingsHandler().getOrder(); String userAgent = curi.getUserAgent(); if (userAgent == null) { userAgent = order.getUserAgent(curi); } method.setRequestHeader("User-Agent", userAgent); method.setRequestHeader("From", order.getFrom(curi)); // Set retry handler. method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler()); final long maxLength = getMaxLength(curi); if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) { method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1))); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) { method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue() && (curi.getViaContext() == null || !Link.PREREQ_MISC.equals(curi.getViaContext().toString()))) { // RFC2616 says no referer header if referer is https and the url // is not String via = curi.flattenVia(); if (via != null && via.length() > 0 && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) { method.setRequestHeader(REFERER, via); } } if (!curi.isPrerequisite()) { setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since"); setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match"); } // TODO: What happens if below method adds a header already // added above: e.g. Connection, Range, or Referer? setAcceptHeaders(curi, method); HostConfiguration config = new HostConfiguration(http.getHostConfiguration()); configureProxy(curi, config); configureBindAddress(curi, config); return config; }
From source file:org.archive.crawler.fetcher.OptimizeFetchHTTP.java
/** * Configure the HttpMethod setting options and headers. * * @param curi CrawlURI from which we pull configuration. * @param method The Method to configure. * @return HostConfiguration copy customized for this CrawlURI *//*from w w w. j a v a 2 s . c o m*/ protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); // // set soTimeout // method.getParams().setSoTimeout( // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) // .intValue()); // Set cookie policy. method.getParams() .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue()) ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); // Use only HTTP/1.0 (to avoid receiving chunked responses) method.getParams().setVersion(HttpVersion.HTTP_1_0); CrawlOrder order = getSettingsHandler().getOrder(); String userAgent = curi.getUserAgent(); if (userAgent == null) { userAgent = order.getUserAgent(curi); } method.setRequestHeader("User-Agent", userAgent); method.setRequestHeader("From", order.getFrom(curi)); // Set retry handler. method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler()); final long maxLength = getMaxLength(curi); if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) { method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1))); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) { method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue() && (curi.getViaContext() == null || !Link.PREREQ_MISC.equals(curi.getViaContext().toString()))) { // RFC2616 says no referer header if referer is https and the url // is not String via = curi.flattenVia(); if (via != null && via.length() > 0 && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) { method.setRequestHeader(REFERER, via); } } /*if(!curi.isPrerequisite() && curi.containsKey(URLInfo.MODIFY_TIME) && (Boolean)getUncheckedAttribute(curi, ATTR_SEND_IF_MODIFIED_SINCE)) { long modifyTime = curi.getLong(URLInfo.MODIFY_TIME); if (modifyTime != 0) { Date date = new Date(modifyTime); method.setRequestHeader("If-Modified-Since", date.toString()); logger.debug(curi.getUURI().toString() + " send header modifyTime:" + date.toGMTString()); } setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since"); setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match"); }*/ // TODO: What happens if below method adds a header already // added above: e.g. Connection, Range, or Referer? setAcceptHeaders(curi, method); HttpClient http = getClient(); HostConfiguration config = new HostConfiguration(http.getHostConfiguration()); configureProxy(curi, config); configureBindAddress(curi, config); return config; }
From source file:org.attribyte.api.http.impl.commons.Commons3Client.java
@Override public Response send(Request request, RequestOptions options) throws IOException { HttpMethod method = null; switch (request.getMethod()) { case GET://from w ww. j av a 2 s.com method = new GetMethod(request.getURI().toString()); method.setFollowRedirects(options.followRedirects); break; case DELETE: method = new DeleteMethod(request.getURI().toString()); break; case HEAD: method = new HeadMethod(request.getURI().toString()); method.setFollowRedirects(options.followRedirects); break; case POST: method = new PostMethod(request.getURI().toString()); if (request.getBody() != null) { ByteArrayRequestEntity requestEntity = new ByteArrayRequestEntity(request.getBody().toByteArray()); ((EntityEnclosingMethod) method).setRequestEntity(requestEntity); } else { PostMethod postMethod = (PostMethod) method; Collection<Parameter> parameters = request.getParameters(); for (Parameter parameter : parameters) { String[] values = parameter.getValues(); for (String value : values) { postMethod.addParameter(parameter.getName(), value); } } } break; case PUT: method = new PutMethod(request.getURI().toString()); if (request.getBody() != null) { ByteArrayRequestEntity requestEntity = new ByteArrayRequestEntity(request.getBody().toByteArray()); ((EntityEnclosingMethod) method).setRequestEntity(requestEntity); } break; } if (userAgent != null && Strings.isNullOrEmpty(request.getHeaderValue("User-Agent"))) { method.setRequestHeader("User-Agent", userAgent); } Collection<Header> headers = request.getHeaders(); for (Header header : headers) { String[] values = header.getValues(); for (String value : values) { method.setRequestHeader(header.getName(), value); } } int responseCode; InputStream is = null; try { responseCode = httpClient.executeMethod(method); is = method.getResponseBodyAsStream(); if (is != null) { byte[] body = Request.bodyFromInputStream(is, options.maxResponseBytes); ResponseBuilder builder = new ResponseBuilder(); builder.setStatusCode(responseCode); builder.addHeaders(getMap(method.getResponseHeaders())); return builder.setBody(body.length != 0 ? body : null).create(); } else { ResponseBuilder builder = new ResponseBuilder(); builder.setStatusCode(responseCode); builder.addHeaders(getMap(method.getResponseHeaders())); return builder.create(); } } finally { if (is != null) { try { is.close(); } catch (IOException ioe) { //Ignore } } method.releaseConnection(); } }
From source file:org.bibsonomy.util.WebUtils.java
/** * Reads from a URL and writes the content into a string. * //from ww w . j a v a 2 s .c om * @param url * @param cookie * @param postData * @param visitBefore * * @return String which holds the page content. * * @throws IOException */ public static String getContentAsString(final String url, final String cookie, final String postData, final String visitBefore) throws IOException { if (present(visitBefore)) { /* * visit URL to get cookies if needed */ client.executeMethod(new GetMethod(visitBefore)); } final HttpMethod method; if (present(postData)) { /* * do a POST request */ final List<NameValuePair> data = new ArrayList<NameValuePair>(); for (final String s : postData.split(AMP_SIGN)) { final String[] p = s.split(EQUAL_SIGN); if (p.length != 2) { continue; } data.add(new NameValuePair(p[0], p[1])); } method = new PostMethod(url); ((PostMethod) method).setRequestBody(data.toArray(new NameValuePair[data.size()])); } else { /* * do a GET request */ method = new GetMethod(url); method.setFollowRedirects(true); } /* * set cookie */ if (present(cookie)) { method.setRequestHeader(COOKIE_HEADER_NAME, cookie); } /* * do request */ final int status = client.executeMethod(method); if (status != HttpStatus.SC_OK) { throw new IOException(url + " returns: " + status); } /* * FIXME: check content type header to ensure that we only read textual * content (and not a PDF, radio stream or DVD image ...) */ /* * collect response */ final String charset = extractCharset(method.getResponseHeader(CONTENT_TYPE_HEADER_NAME).getValue()); final StringBuilder content = inputStreamToStringBuilder(method.getResponseBodyAsStream(), charset); method.releaseConnection(); final String string = content.toString(); if (string.length() > 0) { return string; } return null; }
From source file:org.cancergrid.ws.util.HttpContentReader.java
public static String getHttpContent(String httpUrl, String query, Method method) { LOG.debug("getHttpContent(httpUrl): " + httpUrl); LOG.debug("getHttpContent(query): " + query); LOG.debug("getHttpContent(method): " + method); HttpMethod httpMethod = null; if (httpUrl.contains("&")) { httpUrl = httpUrl.replace("&", "&"); }/*from w w w . ja v a 2 s . co m*/ if (query != null && query.length() > 0 && query.startsWith("?") && query.contains("&")) { query = query.replace("&", "&"); } try { //LOG.debug("Querying: " + httpUrl); if (method == Method.GET) { httpMethod = new GetMethod(httpUrl); if (query != null && query.length() > 0) { httpMethod.setQueryString(query); } } else if (method == Method.POST) { httpMethod = new PostMethod(httpUrl); if (query != null && query.length() > 0) { RequestEntity entity = new StringRequestEntity(query, "text/xml", "UTF-8"); ((PostMethod) httpMethod).setRequestEntity(entity); } } httpMethod.setFollowRedirects(true); httpMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); Protocol.registerProtocol("https", new Protocol("https", new org.apache.commons.httpclient.contrib.ssl.EasySSLProtocolSocketFactory(), 443)); HttpClient client = new HttpClient(); int statusCode = client.executeMethod(httpMethod); if (statusCode != HttpStatus.SC_OK) { LOG.error("Method failed: " + httpMethod.getStatusLine()); LOG.error("Error querying: " + httpMethod.getURI().toString()); throw new Exception("Method failed: " + httpMethod.getStatusLine()); } byte[] responseBody = httpMethod.getResponseBody(); return new String(responseBody, "UTF-8"); } catch (HttpException e) { LOG.error("Fatal protocol violation: " + e.getMessage()); } catch (IOException e) { LOG.error("Fatal transport error: " + e.getMessage()); } catch (Exception e) { LOG.error(e.getMessage()); } finally { httpMethod.releaseConnection(); } return null; }
From source file:org.codeartisans.proxilet.Proxilet.java
/** * Executes the {@link HttpMethod} passed in and sends the proxy response back to the client via the given * {@link HttpServletResponse}./* ww w . java2 s .c o m*/ * * @param httpMethodProxyRequest An object representing the proxy request to be made * @param httpServletResponse An object by which we can send the proxied response back to the client * @throws IOException Can be thrown by the {@link HttpClient}.executeMethod * @throws ServletException Can be thrown to indicate that another error has occurred */ private void executeProxyRequest(HttpMethod httpMethodProxyRequest, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException { // Create a default HttpClient HttpClient httpClient; httpClient = createClientWithLogin(); httpMethodProxyRequest.setFollowRedirects(false); // Execute the request int intProxyResponseCode = httpClient.executeMethod(httpMethodProxyRequest); // String response = httpMethodProxyRequest.getResponseBodyAsString(); // Check if the proxy response is a redirect // The following code is adapted from org.tigris.noodle.filters.CheckForRedirect // Hooray for open source software if (intProxyResponseCode >= HttpServletResponse.SC_MULTIPLE_CHOICES /* 300 */ && intProxyResponseCode < HttpServletResponse.SC_NOT_MODIFIED /* 304 */ ) { String stringStatusCode = Integer.toString(intProxyResponseCode); String stringLocation = httpMethodProxyRequest.getResponseHeader(HEADER_LOCATION).getValue(); if (stringLocation == null) { throw new ServletException("Received status code: " + stringStatusCode + " but no " + HEADER_LOCATION + " header was found in the response"); } // Modify the redirect to go to this proxy servlet rather that the proxied host String stringMyHostName = httpServletRequest.getServerName(); if (httpServletRequest.getServerPort() != 80) { stringMyHostName += ":" + httpServletRequest.getServerPort(); } stringMyHostName += httpServletRequest.getContextPath(); if (followRedirects) { httpServletResponse .sendRedirect(stringLocation.replace(getProxyHostAndPort() + proxyPath, stringMyHostName)); return; } } else if (intProxyResponseCode == HttpServletResponse.SC_NOT_MODIFIED) { // 304 needs special handling. See: // http://www.ics.uci.edu/pub/ietf/http/rfc1945.html#Code304 // We get a 304 whenever passed an 'If-Modified-Since' // header and the data on disk has not changed; server // responds w/ a 304 saying I'm not going to send the // body because the file has not changed. httpServletResponse.setIntHeader(HEADER_CONTENT_LENGTH, 0); httpServletResponse.setStatus(HttpServletResponse.SC_NOT_MODIFIED); return; } // Pass the response code back to the client httpServletResponse.setStatus(intProxyResponseCode); // Pass response headers back to the client Header[] headerArrayResponse = httpMethodProxyRequest.getResponseHeaders(); for (Header header : headerArrayResponse) { if (header.getName().equals("Transfer-Encoding") && header.getValue().equals("chunked") || header.getName().equals("Content-Encoding") && header.getValue().equals("gzip")) { // proxy servlet does not support chunked encoding } else { httpServletResponse.setHeader(header.getName(), header.getValue()); } } List<Header> responseHeaders = Arrays.asList(headerArrayResponse); // FIXME We should handle both String and bytes response in the same way: String response = null; byte[] bodyBytes = null; if (isBodyParameterGzipped(responseHeaders)) { LOGGER.trace("GZipped: true"); if (!followRedirects && intProxyResponseCode == HttpServletResponse.SC_MOVED_TEMPORARILY) { response = httpMethodProxyRequest.getResponseHeader(HEADER_LOCATION).getValue(); httpServletResponse.setStatus(HttpServletResponse.SC_OK); intProxyResponseCode = HttpServletResponse.SC_OK; httpServletResponse.setHeader(HEADER_LOCATION, response); httpServletResponse.setContentLength(response.length()); } else { bodyBytes = ungzip(httpMethodProxyRequest.getResponseBody()); httpServletResponse.setContentLength(bodyBytes.length); } } if (httpServletResponse.getContentType() != null && httpServletResponse.getContentType().contains("text")) { LOGGER.trace("Received status code: {} Response: {}", intProxyResponseCode, response); } else { LOGGER.trace("Received status code: {} [Response is not textual]", intProxyResponseCode); } // Send the content to the client if (response != null) { httpServletResponse.getWriter().write(response); } else if (bodyBytes != null) { httpServletResponse.getOutputStream().write(bodyBytes); } else { IOUtils.copy(httpMethodProxyRequest.getResponseBodyAsStream(), httpServletResponse.getOutputStream()); } }
From source file:org.codehaus.wadi.web.impl.CommonsHttpProxy.java
protected void doProxy(URI uri, WebInvocation context) throws ProxyingException { HttpServletRequest hreq = context.getHreq(); HttpServletResponse hres = context.getHres(); long startTime = System.currentTimeMillis(); String m = hreq.getMethod();/*ww w. java 2 s . c o m*/ Class clazz = (Class) _methods.get(m); if (clazz == null) { throw new IrrecoverableException("unsupported http method: " + m); } HttpMethod hm = null; try { hm = (HttpMethod) clazz.newInstance(); } catch (Exception e) { throw new IrrecoverableException("could not create HttpMethod instance", e); // should never happen } String requestURI = getRequestURI(hreq); hm.setPath(requestURI); String queryString = hreq.getQueryString(); if (queryString != null) { hm.setQueryString(queryString); requestURI += queryString; } hm.setFollowRedirects(false); //hm.setURI(new URI(uri)); hm.setStrictMode(false); // check connection header String connectionHdr = hreq.getHeader("Connection"); // TODO - what if there are multiple values ? if (connectionHdr != null) { connectionHdr = connectionHdr.toLowerCase(); if (connectionHdr.equals("keep-alive") || connectionHdr.equals("close")) connectionHdr = null; // TODO ?? } // copy headers boolean xForwardedFor = false; boolean hasContent = false; int contentLength = 0; Enumeration enm = hreq.getHeaderNames(); while (enm.hasMoreElements()) { // TODO could be better than this! - using javax.servlet ? String hdr = (String) enm.nextElement(); String lhdr = hdr.toLowerCase(); if (_DontProxyHeaders.contains(lhdr)) continue; if (connectionHdr != null && connectionHdr.indexOf(lhdr) >= 0) continue; if ("content-length".equals(lhdr)) { try { contentLength = hreq.getIntHeader(hdr); hasContent = contentLength > 0; } catch (NumberFormatException e) { if (_log.isWarnEnabled()) _log.warn("bad Content-Length header value: " + hreq.getHeader(hdr), e); } } if ("content-type".equals(lhdr)) { hasContent = true; } Enumeration vals = hreq.getHeaders(hdr); while (vals.hasMoreElements()) { String val = (String) vals.nextElement(); if (val != null) { hm.addRequestHeader(hdr, val); // if (_log.isInfoEnabled()) _log.info("Request " + hdr + ": " + val); xForwardedFor |= "X-Forwarded-For".equalsIgnoreCase(hdr); // why is this not in the outer loop ? } } } // cookies... // although we copy cookie headers into the request abover - commons-httpclient thinks it knows better and strips them out before sending. // we have to explicitly use their interface to add the cookies - painful... // DOH! - an org.apache.commons.httpclient.Cookie is NOT a // javax.servlet.http.Cookie - and it looks like the two don't // map onto each other without data loss... HttpState state = new HttpState(); javax.servlet.http.Cookie[] cookies = hreq.getCookies(); if (cookies != null) { for (int i = 0; i < cookies.length; i++) { javax.servlet.http.Cookie c = cookies[i]; String domain = c.getDomain(); if (domain == null) { domain = hreq.getServerName(); // TODO - tmp test // _log.warn("defaulting cookie domain"); } // domain=null; String cpath = c.getPath(); if (cpath == null) { cpath = hreq.getContextPath(); // fix for Jetty // _log.warn("defaulting cookie path"); } //if (_log.isTraceEnabled()) _log.trace("PATH: value="+path+" length="+(path==null?0:path.length())); Cookie cookie = new Cookie(domain, c.getName(), c.getValue(), cpath, c.getMaxAge(), c.getSecure()); // TODO - sort out domain //if (_log.isTraceEnabled()) _log.trace("Cookie: "+cookie.getDomain()+","+ cookie.getName()+","+ cookie.getValue()+","+ cookie.getPath()+","+ cookie.getExpiryDate()+","+ cookie.getSecure()); state.addCookie(cookie); //if (_log.isTraceEnabled()) _log.trace("Cookie: "+cookie.toString()); } } // Proxy headers hm.addRequestHeader("Via", "1.1 " + hreq.getLocalName() + ":" + hreq.getLocalPort() + " \"WADI\""); if (!xForwardedFor) hm.addRequestHeader("X-Forwarded-For", hreq.getRemoteAddr()); // Max-Forwards... // a little bit of cache control // String cache_control = hreq.getHeader("Cache-Control"); // if (cache_control != null && (cache_control.indexOf("no-cache") >= 0 || cache_control.indexOf("no-store") >= 0)) // httpMethod.setUseCaches(false); // customize Connection // uc.setDoInput(true); int client2ServerTotal = 0; if (hasContent) { // uc.setDoOutput(true); try { if (hm instanceof EntityEnclosingMethod) ((EntityEnclosingMethod) hm).setRequestBody(hreq.getInputStream()); // TODO - do we need to close response stream at end... ? } catch (IOException e) { throw new IrrecoverableException("could not pss request input across proxy", e); } } try { HttpClient client = new HttpClient(); HostConfiguration hc = new HostConfiguration(); //String host=location.getAddress().getHostAddress(); // inefficient - but stops httpclient from rejecting half our cookies... String host = uri.getHost(); hc.setHost(host, uri.getPort()); client.executeMethod(hc, hm, state); } catch (IOException e) // TODO { _log.warn("problem proxying connection:", e); } InputStream fromServer = null; // handler status codes etc. int code = 502; // String message="Bad Gateway: could not read server response code or message"; code = hm.getStatusCode(); // IOException // message=hm.getStatusText(); // IOException hres.setStatus(code); // hres.setStatus(code, message); - deprecated... try { fromServer = hm.getResponseBodyAsStream(); // IOException } catch (IOException e) { _log.warn("problem acquiring http client output", e); } // clear response defaults. hres.setHeader("Date", null); hres.setHeader("Server", null); // set response headers // TODO - is it a bug in Jetty that I have to start my loop at 1 ? or that key[0]==null ? // Try this inside Tomcat... Header[] headers = hm.getResponseHeaders(); for (int i = 0; i < headers.length; i++) { String h = headers[i].toExternalForm(); int index = h.indexOf(':'); String key = h.substring(0, index).trim().toLowerCase(); String val = h.substring(index + 1, h.length()).trim(); if (val != null && !_DontProxyHeaders.contains(key)) { hres.addHeader(key, val); // if (_log.isInfoEnabled()) _log.info("Response: "+key+" - "+val); } } hres.addHeader("Via", "1.1 (WADI)"); // copy server->client int server2ClientTotal = 0; if (fromServer != null) { try { OutputStream toClient = hres.getOutputStream();// IOException server2ClientTotal += copy(fromServer, toClient, 8192);// IOException } catch (IOException e) { _log.warn("problem proxying server response back to client", e); } finally { try { fromServer.close(); } catch (IOException e) { // well - we did our best... _log.warn("problem closing server response stream", e); } } } long endTime = System.currentTimeMillis(); long elapsed = endTime - startTime; if (_log.isDebugEnabled()) { _log.debug("in:" + client2ServerTotal + ", out:" + server2ClientTotal + ", status:" + code + ", time:" + elapsed + ", uri:" + uri); } }
From source file:org.eclipsetrader.borsaitalia.internal.core.BackfillConnector.java
@Override public IOHLC[] backfillHistory(IFeedIdentifier identifier, Date from, Date to, TimeSpan timeSpan) { String code = identifier.getSymbol(); String isin = null;//from w w w . j a va2 s.co m IFeedProperties properties = (IFeedProperties) identifier.getAdapter(IFeedProperties.class); if (properties != null) { if (properties.getProperty(Activator.PROP_ISIN) != null) { isin = properties.getProperty(Activator.PROP_ISIN); } if (properties.getProperty(Activator.PROP_CODE) != null) { code = properties.getProperty(Activator.PROP_CODE); } } if (code == null || isin == null) { return null; } String period = String.valueOf(timeSpan.getLength()) + (timeSpan.getUnits() == Units.Minutes ? "MIN" : "DAY"); //$NON-NLS-1$ //$NON-NLS-2$ List<OHLC> list = new ArrayList<OHLC>(); try { HttpMethod method = new GetMethod("http://" + host + "/scripts/cligipsw.dll"); //$NON-NLS-1$ //$NON-NLS-2$ method.setQueryString(new NameValuePair[] { new NameValuePair("app", "tic_d"), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("action", "dwnld4push"), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("cod", code), //$NON-NLS-1$ new NameValuePair("codneb", isin), //$NON-NLS-1$ new NameValuePair("req_type", "GRAF_DS"), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("ascii", "1"), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("form_id", ""), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("period", period), //$NON-NLS-1$ new NameValuePair("From", new SimpleDateFormat("yyyyMMdd000000").format(from)), //$NON-NLS-1$ //$NON-NLS-2$ new NameValuePair("To", new SimpleDateFormat("yyyyMMdd000000").format(to)), //$NON-NLS-1$ //$NON-NLS-2$ }); method.setFollowRedirects(true); HttpClient client = new HttpClient(); client.getHttpConnectionManager().getParams().setConnectionTimeout(5000); client.executeMethod(method); BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); String inputLine = in.readLine(); if (inputLine.startsWith("@")) { //$NON-NLS-1$ while ((inputLine = in.readLine()) != null) { if (inputLine.startsWith("@") || inputLine.length() == 0) { //$NON-NLS-1$ continue; } try { String[] item = inputLine.split("\\|"); //$NON-NLS-1$ OHLC bar = new OHLC(df.parse(item[0]), nf.parse(item[1]).doubleValue(), nf.parse(item[2]).doubleValue(), nf.parse(item[3]).doubleValue(), nf.parse(item[4]).doubleValue(), nf.parse(item[5]).longValue()); list.add(bar); } catch (Exception e) { Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error parsing data: " + inputLine, e); //$NON-NLS-1$ Activator.getDefault().getLog().log(status); } } } else { Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, NLS.bind("Unexpected response from {0}: {1}", new Object[] { //$NON-NLS-1$ method.getURI().toString(), inputLine }), null); Activator.getDefault().getLog().log(status); } in.close(); } catch (Exception e) { Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error reading data", e); //$NON-NLS-1$ Activator.getDefault().getLog().log(status); } return list.toArray(new IOHLC[list.size()]); }
From source file:org.eclipsetrader.directa.internal.core.connector.StreamingConnector.java
protected void fetchLatestBookSnapshot(String[] sTit) { Hashtable<String, String[]> hashtable = new Hashtable<String, String[]>(); try {/*from w ww . ja v a 2 s .c om*/ HttpMethod method = createMethod(sTit, "t", streamingServer, WebConnector.getInstance().getUrt(), //$NON-NLS-1$ WebConnector.getInstance().getPrt()); method.setFollowRedirects(true); HttpClient client = new HttpClient(); setupProxy(client, streamingServer); client.executeMethod(method); BufferedReader bufferedreader = new BufferedReader( new InputStreamReader(method.getResponseBodyAsStream())); String s5; while ((s5 = bufferedreader.readLine()) != null) { String[] campo = s5.split("\\;"); //$NON-NLS-1$ if (campo.length != 0) { hashtable.put(campo[0], campo); } } } catch (Exception e) { Status status = new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error reading snapshot data", e); //$NON-NLS-1$ Activator.log(status); } for (String symbol : sTit) { String sVal[] = hashtable.get(symbol); if (sVal == null) { continue; } FeedSubscription subscription = symbolSubscriptions.get(symbol); if (subscription == null) { continue; } try { Object oldValue = subscription.getBook(); IBookEntry[] bidEntry = new IBookEntry[20]; IBookEntry[] askEntry = new IBookEntry[20]; for (int x = 0, k = 9; x < 20; x++, k += 6) { bidEntry[x] = new BookEntry(null, Double.parseDouble(sVal[k + 2]), Long.parseLong(sVal[k + 1]), Long.parseLong(sVal[k]), null); askEntry[x] = new BookEntry(null, Double.parseDouble(sVal[k + 5]), Long.parseLong(sVal[k + 4]), Long.parseLong(sVal[k + 3]), null); } Object newValue = new org.eclipsetrader.core.feed.Book(bidEntry, askEntry); subscription.setBook((IBook) newValue); subscription.addDelta(new QuoteDelta(subscription.getIdentifier(), oldValue, newValue)); } catch (Exception e) { Activator.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, 0, "Error reading snapshot data", e)); //$NON-NLS-1$ } } wakeupNotifyThread(); }