List of usage examples for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER
String RETRY_HANDLER
To view the source code for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER.
Click Source Link
From source file:com.xmlcalabash.library.HttpRequest.java
private void doPutOrPostMultipart(EntityEnclosingMethod method, XdmNode multipart) { // The Apache HTTP libraries just don't handle this case...we treat it as a "single part" // and build the body ourselves, using the boundaries etc. // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); // Check for consistency of content-type contentType = multipart.getAttributeValue(_content_type); if (contentType == null) { contentType = "multipart/mixed"; }/*from w w w . ja v a2 s .c o m*/ if (headerContentType != null && !headerContentType.equals(contentType.toLowerCase())) { throw XProcException.stepError(20); } if (!contentType.startsWith("multipart/")) { throw new UnsupportedOperationException("Multipart content-type must be multipart/..."); } for (Header header : headers) { method.addRequestHeader(header); } String boundary = multipart.getAttributeValue(_boundary); if (boundary == null) { throw new XProcException(step.getNode(), "A boundary value must be specified on c:multipart"); } if (boundary.startsWith("--")) { throw XProcException.stepError(2); } String q = "\""; if (boundary.contains(q)) { q = "'"; } if (boundary.contains(q)) { q = ""; } String multipartContentType = contentType + "; boundary=" + q + boundary + q; // FIXME: This sucks rocks. I want to write the data to be posted, not provide some way to read it MessageBytes byteContent = new MessageBytes(); byteContent.append("This is a multipart message.\r\n"); //String postContent = "This is a multipart message.\r\n"; for (XdmNode body : new RelevantNodes(runtime, multipart, Axis.CHILD)) { if (!XProcConstants.c_body.equals(body.getNodeName())) { throw new XProcException(step.getNode(), "A c:multipart may only contain c:body elements."); } String bodyContentType = body.getAttributeValue(_content_type); if (bodyContentType == null) { throw new XProcException(step.getNode(), "Content-type on c:body is required."); } String bodyId = body.getAttributeValue(_id); String bodyDescription = body.getAttributeValue(_description); String bodyDisposition = body.getAttributeValue(_disposition); String bodyCharset = HttpUtils.getCharset(bodyContentType); if (bodyContentType.contains(";")) { int pos = bodyContentType.indexOf(";"); bodyContentType = bodyContentType.substring(0, pos); } String bodyEncoding = body.getAttributeValue(_encoding); if (bodyEncoding != null && !"base64".equals(bodyEncoding)) { throw new UnsupportedOperationException("The '" + bodyEncoding + "' encoding is not supported"); } if (bodyCharset != null) { bodyContentType += "; charset=" + bodyCharset; } else { // Is utf-8 the right default? What about the image/ case? bodyContentType += "; charset=utf-8"; } //postContent += "--" + boundary + "\r\n"; //postContent += "Content-Type: " + bodyContentType + "\r\n"; byteContent.append("--" + boundary + "\r\n"); byteContent.append("Content-Type: " + bodyContentType + "\r\n"); if (bodyDescription != null) { //postContent += "Content-Description: " + bodyDescription + "\r\n"; byteContent.append("Content-Description: " + bodyDescription + "\r\n"); } if (bodyId != null) { //postContent += "Content-ID: " + bodyId + "\r\n"; byteContent.append("Content-ID: " + bodyId + "\r\n"); } if (bodyDisposition != null) { //postContent += "Content-Disposition: " + bodyDisposition + "\r\n"; byteContent.append("Content-Disposition: " + bodyDisposition + "\r\n"); } if (bodyEncoding != null) { //postContent += "Content-Transfer-Encoding: " + bodyEncoding + "\r\n"; if (encodeBinary) { byteContent.append("Content-Transfer-Encoding: " + bodyEncoding + "\r\n"); } } //postContent += "\r\n"; byteContent.append("\r\n"); try { if (xmlContentType(bodyContentType)) { Serializer serializer = makeSerializer(); Vector<XdmNode> content = new Vector<XdmNode>(); XdmSequenceIterator iter = body.axisIterator(Axis.CHILD); while (iter.hasNext()) { XdmNode node = (XdmNode) iter.next(); content.add(node); } // FIXME: set serializer properties appropriately! StringWriter writer = new StringWriter(); serializer.setOutputWriter(writer); S9apiUtils.serialize(runtime, content, serializer); writer.close(); //postContent += writer.toString(); byteContent.append(writer.toString()); } else if (jsonContentType(contentType)) { byteContent.append(XMLtoJSON.convert(body)); } else if (!encodeBinary && "base64".equals(bodyEncoding)) { byte[] decoded = Base64.decode(body.getStringValue()); byteContent.append(decoded, decoded.length); } else { StringWriter writer = new StringWriter(); XdmSequenceIterator iter = body.axisIterator(Axis.CHILD); while (iter.hasNext()) { XdmNode node = (XdmNode) iter.next(); if (node.getNodeKind() != XdmNodeKind.TEXT) { throw XProcException.stepError(28); } writer.write(node.getStringValue()); } writer.close(); //postContent += writer.toString(); byteContent.append(writer.toString()); } //postContent += "\r\n"; byteContent.append("\r\n"); } catch (IOException ioe) { throw new XProcException(ioe); } catch (SaxonApiException sae) { throw new XProcException(sae); } } //postContent += "--" + boundary + "--\r\n"; byteContent.append("--" + boundary + "--\r\n"); ByteArrayRequestEntity requestEntity = new ByteArrayRequestEntity(byteContent.content(), multipartContentType); //StringRequestEntity requestEntity = new StringRequestEntity(postContent, multipartContentType, null); method.setRequestEntity(requestEntity); }
From source file:com.motorola.studio.android.localization.translators.GoogleTranslator.java
/** * Creates an HTTP request with the URL, execute it as a get, and returns * the a string with the result.//from w ww . j a va 2 s . c o m * * @param url * URL to be executed. * @return String with the URL execution result. * @throws IOException * If an exception occurs on transport * @throws HttpException * If an exception occurs on the protocol * @throws Exception * on error. */ protected static String executeHttpGetRequest(final URL url) throws HttpException { // Checking query size due to google policies if (url.toString().length() > MAX_QUERY_SIZE) { throw new HttpException(TranslateNLS.GoogleTranslator_Error_QueryTooBig); } // Try to retrieve proxy configuration to use if necessary IProxyService proxyService = ProxyManager.getProxyManager(); IProxyData proxyData = null; if (proxyService.isProxiesEnabled() || proxyService.isSystemProxiesEnabled()) { Authenticator.setDefault(new ProxyAuthenticator()); String urlStr = url.toString(); if (urlStr.startsWith("https")) { proxyData = proxyService.getProxyData(IProxyData.HTTPS_PROXY_TYPE); StudioLogger.debug(GoogleTranslator.class, "Using https proxy"); //$NON-NLS-1$ } else if (urlStr.startsWith("http")) { proxyData = proxyService.getProxyData(IProxyData.HTTP_PROXY_TYPE); StudioLogger.debug(GoogleTranslator.class, "Using http proxy"); //$NON-NLS-1$ } else { StudioLogger.debug(GoogleTranslator.class, "Not using any proxy"); //$NON-NLS-1$ } } // Creates the http client and the method to be executed HttpClient client = null; client = new HttpClient(); // If there is proxy data, work with it if (proxyData != null) { if (proxyData.getHost() != null) { // Sets proxy host and port, if any client.getHostConfiguration().setProxy(proxyData.getHost(), proxyData.getPort()); } if (proxyData.getUserId() != null && proxyData.getUserId().trim().length() > 0) { // Sets proxy user and password, if any Credentials cred = new UsernamePasswordCredentials(proxyData.getUserId(), proxyData.getPassword() == null ? "" : proxyData.getPassword()); //$NON-NLS-1$ client.getState().setProxyCredentials(AuthScope.ANY, cred); } } // Creating the method to be executed, the URL at this point is enough // because it is complete GetMethod method = new GetMethod(url.toString()); // Set method to be retried three times in case of error method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(RETRIES, false)); method.setRequestHeader(REFERER_HEADER, REFERER_SITE); // Set the connection timeout client.getHttpConnectionManager().getParams().setConnectionTimeout(new Integer(TIMEOUT)); String result = ""; //$NON-NLS-1$ try { // Execute the method. int statusCode; try { statusCode = client.executeMethod(method); result = method.getResponseBodyAsString(MAX_SIZE); } catch (IOException e) { throw new HttpException(TranslateNLS.GoogleTranslator_Error_CannotConnectToServer + e.getMessage()); } checkStatusCode(statusCode, result); // Unescape any possible unicode char result = unescapeUnicode(result); // Unescape any possible HTML sequence result = unescapeHTML(result); } finally { // Release the connection. method.releaseConnection(); } return result; }
From source file:com.silverwrist.venice.std.TrackbackManager.java
/** * Loads the HTTP content at the specified URL, scans it for RDF description blocks, and adds those blocks * as {@link com.silverwrist.venice.std.TrackbackItem TrackbackItem}s to our internal cache. Uses modification * detection to keep from reloading a page unless necessary. * * @param url The URL of the resource to be loaded. * @param attrs The attributes of the specified page; if this is <code>null</code>, we'll check the page * cache for the right attributes. * @return <code>true</code> if the page data was loaded and scanned for trackback items; <code>false</code> * if no data was loaded (because it was not modified since the last time we loaded it, for instance). * @exception com.silverwrist.venice.except.TrackbackException If there was an error loading or interpreting * the page data./*from w w w . j a v a 2 s. c o m*/ */ private synchronized boolean load(URL url, PageAttributes attrs) throws TrackbackException { if (attrs == null) attrs = (PageAttributes) (m_page_cache.get(url)); // Create the GET method and set its headers. String s = url.toString(); int x = s.lastIndexOf('#'); if (x >= 0) s = s.substring(0, x); GetMethod getter = new GetMethod(s); HttpMethodParams params = getter.getParams(); getter.setDoAuthentication(false); getter.setFollowRedirects(true); getter.setRequestHeader("User-Agent", USER_AGENT); getter.setRequestHeader("Accept", "text/*"); getter.setRequestHeader("Accept-Encoding", "identity"); params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); getter.setParams(params); boolean get_resp = false; PageAttributes newattrs = null; ContentType ctype = null; byte[] rawdata = null; try { // set the Last-Modified date as an If-Modified-Since header on the request java.util.Date lmod = null; if (attrs != null) lmod = attrs.getLastModified(); if (lmod != null) getter.setRequestHeader("If-Modified-Since", s_httpdate_format.format(lmod)); // execute the Get method! int rc = m_http_client.executeMethod(getter); get_resp = true; if ((lmod != null) && (rc == HttpStatus.SC_NOT_MODIFIED)) return false; // we were not modified if (rc == HttpStatus.SC_NO_CONTENT) return false; // there's no content there if (rc != HttpStatus.SC_OK) // this is farked! throw new TrackbackException("GET of " + url + " returned " + rc); // Get the new page attributes and save them off. newattrs = new PageAttributes(getter); m_page_cache.put(url, newattrs); // Get the Content-Type header and see if it's valid. Header hdr = getter.getResponseHeader("Content-Type"); if (hdr != null) s = hdr.getValue(); else s = "text/plain"; // necessary assumption ctype = new ContentType(s); if (!(ctype.getPrimaryType().equals("text"))) throw new TrackbackException("URL " + url + " does not point to a text-based resource"); // Load the resource in as byte data; we will determine the right character set for it later. rawdata = getter.getResponseBody(); get_resp = false; } // end try catch (IOException e) { // IO error getting the page throw new TrackbackException("I/O error retrieving " + url + ": " + e.getMessage(), e); } // end catch catch (javax.mail.internet.ParseException e) { // translate into TrackbackException throw new TrackbackException("invalid Content-Type received for URL " + url, e); } // end catch finally { // release the connection if possible try { // need to get the message body if (get_resp) getter.getResponseBody(); } // end try catch (IOException e) { // ignore these } // end catch getter.releaseConnection(); } // end finally // make a first guess at the charset from the HTTP header Content-Type String cset = ctype.getParameter("charset"); if (cset == null) cset = "US-ASCII"; String content = null; try { // interpret the content content = new String(rawdata, cset); } // end try catch (UnsupportedEncodingException e) { // fall back and try just using US-ASCII cset = null; try { // interpret the content content = new String(rawdata, "US-ASCII"); } // end try catch (UnsupportedEncodingException e2) { // can't happen logger.debug("WTF? US-ASCII should damn well be a supported character set!", e2); } // end catch } // end catch // Look for <META HTTP-EQUIV=...> tags in the content. Map http_attrs = extractHttpEquivTags(content); // Try to get a Content-Type attribute from there. s = (String) (http_attrs.get("CONTENT-TYPE")); String cset2 = null; if (s != null) { // look for the content type try { // parse into Content-Type ContentType c = new ContentType(s); if (c.getPrimaryType().equals("text")) cset2 = c.getParameter("charset"); } // end try catch (javax.mail.internet.ParseException e) { // can't get a second Content-Type logger.debug("parse of Content-Type from META tags failed", e); cset2 = null; } // end catch } // end if if ((cset == null) && (cset2 == null)) throw new TrackbackException("unable to determine character set for " + url); if ((cset2 != null) && ((cset == null) || !(cset.equalsIgnoreCase(cset2)))) { // reinterpret content in new character set try { // reinterpret content in new character set s = new String(rawdata, cset2); content = s; // the contents of the HTTP-EQUIV tags may have changed as a result http_attrs = extractHttpEquivTags(content); } // end try catch (UnsupportedEncodingException e) { // just use original character set if (cset == null) throw new TrackbackException("unable to determine character set for " + url); } // end catch } // end if newattrs.updateFromPage(http_attrs); // update the page attributes from the META tag data // Search the page content for RDF blocks. RE m = new RE(s_rdf_start, RE.MATCH_NORMAL); int pos = 0; while (m.match(content, pos)) { // look for the end of this RDF block RE m2 = new RE(getEndRecognizer(m.getParen(1)), RE.MATCH_NORMAL); if (m2.match(content, m.getParenEnd(0))) { // we now have a block to feed to the XML parser try { // run the block through the XML parser InputSource isrc = new InputSource( new StringReader(content.substring(m.getParenStart(0), m2.getParenEnd(0)))); Document doc = m_rdf_parser.parse(isrc); // examine topmost element, which should be rdf:RDF Element root = doc.getDocumentElement(); if (NS_RDF.equals(root.getNamespaceURI()) && (root.getLocalName() != null) && root.getLocalName().equals("RDF")) { // this is most definitely an rdf:RDF node...look for rdf:Description nodes under it NodeList nl = root.getChildNodes(); for (int i = 0; i < nl.getLength(); i++) { // check each node in the list Node n = nl.item(i); if ((n.getNodeType() == Node.ELEMENT_NODE) && NS_RDF.equals(n.getNamespaceURI()) && (n.getLocalName() != null) && n.getLocalName().equals("Description")) { // we've got an rdf:Description node...extract the attributes from it Element elt = (Element) n; try { // look for the item and trackback URLs URL item = null, trackback = null; s = elt.getAttributeNS(NS_DC, "identifier"); if ((s != null) && (s.length() > 0)) item = new URL(s); s = elt.getAttributeNS(NS_TRACKBACK, "ping"); if ((s != null) && (s.length() > 0)) trackback = new URL(s); if ((item != null) && (trackback != null)) { // create the item s = elt.getAttributeNS(NS_DC, "title"); m_item_cache.put(item, new MyTrackbackItem(item, trackback, s, newattrs)); } // end if } // end try catch (MalformedURLException e) { // this means skip this item logger.warn("URL parse failure", e); } // end catch } // end if } // end for } // end if } // end try catch (IOException e) { // disregard this block logger.warn("RDF block parse failure", e); } // end catch catch (SAXException e) { // disregard this block logger.warn("RDF block parse failure", e); } // end catch } // end if // else ignore this possible block pos = m.getParenEnd(0); } // end while return true; }
From source file:com.eucalyptus.imaging.backend.ImagingTaskStateManager.java
private boolean doesManifestExist(final String manifestUrl) throws Exception { // validate urls per EUCA-9144 final UrlValidator urlValidator = new UrlValidator(); if (!urlValidator.isEucalyptusUrl(manifestUrl)) throw new RuntimeException("Manifest's URL is not in the Eucalyptus format: " + manifestUrl); final HttpClient client = new HttpClient(); client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); client.getParams().setParameter(HttpConnectionParams.CONNECTION_TIMEOUT, 10000); client.getParams().setParameter(HttpConnectionParams.SO_TIMEOUT, 30000); GetMethod method = new GetMethod(manifestUrl); String manifest = null;/* ww w . j a va2 s .co m*/ try { // avoid TCP's CLOSE_WAIT method.setRequestHeader("Connection", "close"); client.executeMethod(method); manifest = method.getResponseBodyAsString(ImageConfiguration.getInstance().getMaxManifestSizeBytes()); if (manifest == null) { return false; } else if (manifest.contains("<Code>NoSuchKey</Code>") || manifest.contains("The specified key does not exist")) { return false; } } catch (final Exception ex) { return false; } finally { method.releaseConnection(); } final List<String> partsUrls = getPartsHeadUrl(manifest); for (final String url : partsUrls) { if (!urlValidator.isEucalyptusUrl(url)) throw new RuntimeException("Manifest's URL is not in the Eucalyptus format: " + url); HeadMethod partCheck = new HeadMethod(url); int res = client.executeMethod(partCheck); if (res != HttpStatus.SC_OK) { return false; } } return true; }
From source file:com.eucalyptus.imaging.ImagingTaskStateManager.java
private boolean doesManifestExist(final String manifestUrl) throws Exception { // validate urls per EUCA-9144 final UrlValidator urlValidator = new UrlValidator(); if (!urlValidator.isEucalyptusUrl(manifestUrl)) throw new RuntimeException("Manifest's URL is not in the Eucalyptus format: " + manifestUrl); final HttpClient client = new HttpClient(); client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); client.getParams().setParameter(HttpConnectionParams.CONNECTION_TIMEOUT, 10000); client.getParams().setParameter(HttpConnectionParams.SO_TIMEOUT, 30000); GetMethod method = new GetMethod(manifestUrl); String manifest = null;// ww w .j a va 2 s .c om try { // avoid TCP's CLOSE_WAIT method.setRequestHeader("Connection", "close"); client.executeMethod(method); manifest = method.getResponseBodyAsString(); if (manifest == null) { return false; } else if (manifest.contains("<Code>NoSuchKey</Code>") || manifest.contains("The specified key does not exist")) { return false; } } catch (final Exception ex) { return false; } finally { method.releaseConnection(); } final List<String> partsUrls = getPartsHeadUrl(manifest); for (final String url : partsUrls) { if (!urlValidator.isEucalyptusUrl(url)) throw new RuntimeException("Manifest's URL is not in the Eucalyptus format: " + url); HeadMethod partCheck = new HeadMethod(url); int res = client.executeMethod(partCheck); if (res != HttpStatus.SC_OK) { return false; } } return true; }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
/** * Configure the HttpMethod setting options and headers. * * @param curi CrawlURI from which we pull configuration. * @param method The Method to configure. * @return HostConfiguration copy customized for this CrawlURI *//*from www. ja v a2 s.c o m*/ protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); // // set soTimeout // method.getParams().setSoTimeout( // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) // .intValue()); // Set cookie policy. method.getParams() .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue()) ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); // Use only HTTP/1.0 (to avoid receiving chunked responses) method.getParams().setVersion(HttpVersion.HTTP_1_0); CrawlOrder order = getSettingsHandler().getOrder(); String userAgent = curi.getUserAgent(); if (userAgent == null) { userAgent = order.getUserAgent(curi); } method.setRequestHeader("User-Agent", userAgent); method.setRequestHeader("From", order.getFrom(curi)); // Set retry handler. method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler()); final long maxLength = getMaxLength(curi); if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) { method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1))); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) { method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue()) { // RFC2616 says no referer header if referer is https and the url // is not String via = curi.flattenVia(); if (via != null && via.length() > 0 && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) { method.setRequestHeader(REFERER, via); } } if (!curi.isPrerequisite()) { setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since"); setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match"); } // TODO: What happens if below method adds a header already // added above: e.g. Connection, Range, or Referer? setAcceptHeaders(curi, method); HostConfiguration config = new HostConfiguration(http.getHostConfiguration()); configureProxy(curi, config); configureBindAddress(curi, config); return config; }
From source file:com.twinsoft.convertigo.beans.connectors.SiteClipperConnector.java
private void doProcessRequest(Shuttle shuttle) throws IOException, ServletException, EngineException { shuttle.statisticsTaskID = context.statistics.start(EngineStatistics.GET_DOCUMENT); try {//from ww w . j a v a 2 s .c o m shuttle.sharedScope = context.getSharedScope(); String domain = shuttle.getRequest(QueryPart.host) + shuttle.getRequest(QueryPart.port); Engine.logSiteClipper.trace("(SiteClipperConnector) Prepare the request for the domain " + domain); if (!shouldRewrite(domain)) { Engine.logSiteClipper.info( "(SiteClipperConnector) The domain " + domain + " is not allowed with this connector"); shuttle.response.sendError(HttpServletResponse.SC_FORBIDDEN, "The domain " + domain + " is not allowed with this connector"); return; } String uri = shuttle.getRequest(QueryPart.uri); Engine.logSiteClipper.info("Preparing " + shuttle.request.getMethod() + " " + shuttle.getRequestUrl()); HttpMethod httpMethod = null; XulRecorder xulRecorder = context.getXulRecorder(); if (xulRecorder != null) { httpMethod = shuttle.httpMethod = xulRecorder.getRecord(shuttle.getRequestUrlAndQuery()); } if (httpMethod == null) { try { switch (shuttle.getRequestHttpMethodType()) { case GET: httpMethod = new GetMethod(uri); break; case POST: httpMethod = new PostMethod(uri); ((PostMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case PUT: httpMethod = new PutMethod(uri); ((PutMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case DELETE: httpMethod = new DeleteMethod(uri); break; case HEAD: httpMethod = new HeadMethod(uri); break; case OPTIONS: httpMethod = new OptionsMethod(uri); break; case TRACE: httpMethod = new TraceMethod(uri); break; default: throw new ServletException( "(SiteClipperConnector) unknown http method " + shuttle.request.getMethod()); } httpMethod.setFollowRedirects(false); } catch (Exception e) { throw new ServletException( "(SiteClipperConnector) unexpected exception will building the http method : " + e.getMessage()); } shuttle.httpMethod = httpMethod; SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Request screen class: " + screenClass.getName()); for (String name : Collections .list(GenericUtils.<Enumeration<String>>cast(shuttle.request.getHeaderNames()))) { if (requestHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring request header " + name); } else { String value = shuttle.request.getHeader(name); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying request header " + name + "=" + value); shuttle.setRequestCustomHeader(name, value); } } Engine.logSiteClipper.debug("(SiteClipperConnector) applying request rules for the screenclass " + screenClass.getName()); for (IRequestRule rule : screenClass.getRequestRules()) { if (rule.isEnabled()) { Engine.logSiteClipper .trace("(SiteClipperConnector) applying request rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnRequest(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the request rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled request rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.requestCustomHeaders.entrySet()) { Engine.logSiteClipper.trace("(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); httpMethod.addRequestHeader(header.getKey(), header.getValue()); } String queryString = shuttle.request.getQueryString(); if (queryString != null) { try { // Fake test in order to check query string validity new URI("http://localhost/index?" + queryString, true, httpMethod.getParams().getUriCharset()); } catch (URIException e) { // Bugfix #2103 StringBuffer newQuery = new StringBuffer(); for (String part : RegexpUtils.pattern_and.split(queryString)) { String[] pair = RegexpUtils.pattern_equals.split(part, 2); try { newQuery.append('&') .append(URLEncoder.encode(URLDecoder.decode(pair[0], "UTF-8"), "UTF-8")); if (pair.length > 1) { newQuery.append('=').append( URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8")); } } catch (UnsupportedEncodingException ee) { Engine.logSiteClipper .trace("(SiteClipperConnector) failed to encode query part : " + part); } } queryString = newQuery.length() > 0 ? newQuery.substring(1) : newQuery.toString(); Engine.logSiteClipper.trace("(SiteClipperConnector) re-encode query : " + queryString); } } Engine.logSiteClipper.debug("(SiteClipperConnector) Copying the query string : " + queryString); httpMethod.setQueryString(queryString); // if (context.httpState == null) { // Engine.logSiteClipper.debug("(SiteClipperConnector) Creating new HttpState for context id " + context.contextID); // context.httpState = new HttpState(); // } else { // Engine.logSiteClipper.debug("(SiteClipperConnector) Using HttpState of context id " + context.contextID); // } getHttpState(shuttle); HostConfiguration hostConfiguration = getHostConfiguration(shuttle); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setBooleanParameter("http.connection.stalecheck", true); httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, true)); Engine.logSiteClipper.info("Requesting " + httpMethod.getName() + " " + hostConfiguration.getHostURL() + httpMethod.getURI().toString()); HttpClient httpClient = context.getHttpClient3(shuttle.getHttpPool()); HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, shuttle.getHttpPool()); httpClient.executeMethod(hostConfiguration, httpMethod, context.httpState); } else { Engine.logSiteClipper.info("Retrieve recorded response from Context"); } int status = httpMethod.getStatusCode(); shuttle.processState = ProcessState.response; Engine.logSiteClipper.info("Request terminated with status " + status); shuttle.response.setStatus(status); if (Engine.isStudioMode() && status == HttpServletResponse.SC_OK && shuttle.getResponseMimeType().startsWith("text/")) { fireDataChanged(new ConnectorEvent(this, shuttle.getResponseAsString())); } SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Response screen class: " + screenClass.getName()); if (Engine.isStudioMode()) { Engine.theApp.fireObjectDetected(new EngineEvent(screenClass)); } for (Header header : httpMethod.getResponseHeaders()) { String name = header.getName(); if (responseHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring response header " + name); } else { String value = header.getValue(); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying response header " + name + "=" + value); shuttle.responseCustomHeaders.put(name, value); } } Engine.logSiteClipper.debug( "(SiteClipperConnector) applying response rules for the screenclass " + screenClass.getName()); for (IResponseRule rule : screenClass.getResponseRules()) { if (rule.isEnabled()) { Engine.logSiteClipper.trace("(SiteClipperConnector) applying response rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnResponse(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the response rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled response rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.responseCustomHeaders.entrySet()) { Engine.logSiteClipper.trace( "(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); shuttle.response.addHeader(header.getKey(), header.getValue()); } if (shuttle.postInstructions != null) { JSONArray instructions = new JSONArray(); for (IClientInstruction instruction : shuttle.postInstructions) { try { instructions.put(instruction.getInstruction()); } catch (JSONException e) { Engine.logSiteClipper.error( "(SiteClipperConnector) Failed to add a post instruction due to a JSONException", e); } } String codeToInject = "<script>C8O_postInstructions = " + instructions.toString() + "</script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/jquery.min.js\"></script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/siteclipper.js\"></script>\n"; String content = shuttle.getResponseAsString(); Matcher matcher = HtmlLocation.head_top.matcher(content); String newContent = RegexpUtils.inject(matcher, codeToInject); if (newContent == null) { matcher = HtmlLocation.body_top.matcher(content); newContent = RegexpUtils.inject(matcher, codeToInject); } if (newContent != null) { shuttle.setResponseAsString(newContent); } else { Engine.logSiteClipper.info( "(SiteClipperConnector) Failed to find a head or body tag in the response content"); Engine.logSiteClipper.trace("(SiteClipperConnector) Response content : \"" + content + "\""); } } long nbBytes = 0L; if (shuttle.responseAsString != null && shuttle.responseAsString.hashCode() != shuttle.responseAsStringOriginal.hashCode()) { OutputStream os = shuttle.response.getOutputStream(); switch (shuttle.getResponseContentEncoding()) { case gzip: os = new GZIPOutputStream(os); break; case deflate: os = new DeflaterOutputStream(os, new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true)); break; default: break; } nbBytes = shuttle.responseAsByte.length; IOUtils.write(shuttle.responseAsString, os, shuttle.getResponseCharset()); os.close(); } else { InputStream is = (shuttle.responseAsByte == null) ? httpMethod.getResponseBodyAsStream() : new ByteArrayInputStream(shuttle.responseAsByte); if (is != null) { nbBytes = 0; OutputStream os = shuttle.response.getOutputStream(); int read = is.read(); while (read >= 0) { os.write(read); os.flush(); read = is.read(); nbBytes++; } is.close(); // nbBytes = IOUtils.copyLarge(is, shuttle.response.getOutputStream()); Engine.logSiteClipper .trace("(SiteClipperConnector) Response body copyied (" + nbBytes + " bytes)"); } } shuttle.response.getOutputStream().close(); shuttle.score = getScore(nbBytes); Engine.logSiteClipper .debug("(SiteClipperConnector) Request terminated with a score of " + shuttle.score); } finally { long duration = context.statistics.stop(shuttle.statisticsTaskID); if (context.requestedObject != null) { try { Engine.theApp.billingManager.insertBilling(context, Long.valueOf(duration), Long.valueOf(shuttle.score)); } catch (Exception e) { Engine.logContext.warn("Unable to insert billing ticket (the billing is thus ignored): [" + e.getClass().getName() + "] " + e.getMessage()); } } } }
From source file:nz.co.jsrsolutions.ds3.provider.EodDataEodDataProvider.java
public EodDataEodDataProvider(String url, String username, String password, long timeout) throws EodDataProviderException { this.url = url; this.username = username; this.password = password; this.timeout = timeout; try {/* w w w. j a v a2s . co m*/ HttpMethodParams methodParams = new HttpMethodParams(); DefaultHttpMethodRetryHandler retryHandler = new DefaultHttpMethodRetryHandler(3, false); methodParams.setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); eodDataStub = new DataStub(url); eodDataStub._getServiceClient().getOptions().setTimeOutInMilliSeconds(timeout); eodDataStub._getServiceClient().getOptions().setProperty(HTTPConstants.HTTP_METHOD_PARAMS, methodParams); DataStub.Login loginRequest = new DataStub.Login(); loginRequest.setUsername(username); loginRequest.setPassword(password); // Login DataStub.LoginResponse0 loginResponse0 = eodDataStub.login(loginRequest); DataStub.LOGINRESPONSE loginResponse = loginResponse0.getLoginResult(); if (loginResponse == null) { throw new EodDataProviderException("Failed to authenticate with EOD Data web service."); } token = loginResponse.getToken(); if (token == null || token.isEmpty()) { throw new EodDataProviderException("Failed to authenticate with EOD Data web service."); } logger.info(loginResponse.getMessage()); logger.info(token); logger.info(loginResponse.getDataFormat()); } catch (org.apache.axis2.AxisFault afe) { logger.error(afe.toString()); EodDataProviderException edpe = new EodDataProviderException("Unable to construct EodDataDataProvider"); edpe.initCause(afe); throw edpe; } catch (java.rmi.RemoteException re) { logger.error(re.toString()); EodDataProviderException edpe = new EodDataProviderException("Unable to construct EodDataDataProvider"); edpe.initCause(re); throw edpe; } }
From source file:open.hyperion.nimblestorage.connection.NimbleStorageAPIFactory.java
/** * Initialize HTTP client/*from w w w.j a v a2 s . co m*/ */ public void init() { _log.info("NimbleStorageDriver:NimbleStorageAPIFactory init enter"); _clientMap = new ConcurrentHashMap<String, NimbleStorageAPI>(); HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setDefaultMaxConnectionsPerHost(_maxConnPerHost); params.setMaxTotalConnections(_maxConn); params.setTcpNoDelay(true); params.setConnectionTimeout(_connTimeout); params.setSoTimeout(_socketConnTimeout); _connectionManager = new MultiThreadedHttpConnectionManager(); _connectionManager.setParams(params); _connectionManager.closeIdleConnections(0); // close idle connections immediately HttpClient client = new HttpClient(_connectionManager); client.getParams().setConnectionManagerTimeout(_connManagerTimeout); client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HttpMethodRetryHandler() { @Override public boolean retryMethod(HttpMethod httpMethod, IOException e, int i) { return false; } }); Protocol.registerProtocol("https", new Protocol("https", new NonValidatingSocketFactory(), 8080)); }
From source file:open.hyperion.purestorage.connection.PureStorageAPIFactory.java
/** * Initialize HTTP client// w ww . ja v a2s . c o m */ public void init() { _log.info("PureStorageDriver:PureStorageAPIFactory init enter"); _clientMap = new ConcurrentHashMap<String, PureStorageAPI>(); HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setDefaultMaxConnectionsPerHost(_maxConnPerHost); params.setMaxTotalConnections(_maxConn); params.setTcpNoDelay(true); params.setConnectionTimeout(_connTimeout); params.setSoTimeout(_socketConnTimeout); _connectionManager = new MultiThreadedHttpConnectionManager(); _connectionManager.setParams(params); _connectionManager.closeIdleConnections(0); // close idle connections immediately HttpClient client = new HttpClient(_connectionManager); client.getParams().setConnectionManagerTimeout(_connManagerTimeout); client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HttpMethodRetryHandler() { @Override public boolean retryMethod(HttpMethod httpMethod, IOException e, int i) { return false; } }); Protocol.registerProtocol("https", new Protocol("https", new NonValidatingSocketFactory(), 8080)); }