List of usage examples for org.apache.http.client.utils URLEncodedUtils isEncoded
public static boolean isEncoded(final HttpEntity entity)
application/x-www-form-urlencoded. From source file:com.groupon.odo.bmp.http.BrowserMobHttpClient.java
private BrowserMobHttpResponse execute(BrowserMobHttpRequest req, int depth) { if (depth >= MAX_REDIRECT) { throw new IllegalStateException("Max number of redirects (" + MAX_REDIRECT + ") reached"); }/*from w w w . j a va 2 s.c o m*/ RequestCallback callback = req.getRequestCallback(); HttpRequestBase method = req.getMethod(); String verificationText = req.getVerificationText(); String url = method.getURI().toString(); // save the browser and version if it's not yet been set if (har != null && har.getLog().getBrowser() == null) { Header[] uaHeaders = method.getHeaders("User-Agent"); if (uaHeaders != null && uaHeaders.length > 0) { String userAgent = uaHeaders[0].getValue(); try { // note: this doesn't work for 'Fandango/4.5.1 CFNetwork/548.1.4 Darwin/11.0.0' ReadableUserAgent uai = PARSER.parse(userAgent); String browser = uai.getName(); String version = uai.getVersionNumber().toVersionString(); har.getLog().setBrowser(new HarNameVersion(browser, version)); } catch (Exception e) { LOG.warn("Failed to parse user agent string", e); } } } // process any rewrite requests boolean rewrote = false; String newUrl = url; for (RewriteRule rule : rewriteRules) { Matcher matcher = rule.match.matcher(newUrl); newUrl = matcher.replaceAll(rule.replace); rewrote = true; } if (rewrote) { try { method.setURI(new URI(newUrl)); url = newUrl; } catch (URISyntaxException e) { LOG.warn("Could not rewrite url to %s", newUrl); } } // handle whitelist and blacklist entries int mockResponseCode = -1; synchronized (this) { // guard against concurrent modification of whitelistEntry if (whitelistEntry != null) { boolean found = false; for (Pattern pattern : whitelistEntry.patterns) { if (pattern.matcher(url).matches()) { found = true; break; } } if (!found) { mockResponseCode = whitelistEntry.responseCode; } } } if (blacklistEntries != null) { for (BlacklistEntry blacklistEntry : blacklistEntries) { if (blacklistEntry.pattern.matcher(url).matches()) { mockResponseCode = blacklistEntry.responseCode; break; } } } if (!additionalHeaders.isEmpty()) { // Set the additional headers for (Map.Entry<String, String> entry : additionalHeaders.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); method.removeHeaders(key); method.addHeader(key, value); } } String charSet = "UTF-8"; String responseBody = null; InputStream is = null; int statusCode = -998; long bytes = 0; boolean gzipping = false; boolean contentMatched = true; OutputStream os = req.getOutputStream(); if (os == null) { os = new CappedByteArrayOutputStream(1024 * 1024); // MOB-216 don't buffer more than 1 MB } if (verificationText != null) { contentMatched = false; } // link the object up now, before we make the request, so that if we get cut off (ie: favicon.ico request and browser shuts down) // we still have the attempt associated, even if we never got a response HarEntry entry = new HarEntry(harPageRef); // clear out any connection-related information so that it's not stale from previous use of this thread. RequestInfo.clear(url, entry); entry.setRequest(new HarRequest(method.getMethod(), url, method.getProtocolVersion().getProtocol())); entry.setResponse(new HarResponse(-999, "NO RESPONSE", method.getProtocolVersion().getProtocol())); if (this.har != null && harPageRef != null) { har.getLog().addEntry(entry); } String query = method.getURI().getRawQuery(); if (query != null) { MultiMap<String> params = new MultiMap<String>(); UrlEncoded.decodeTo(query, params, "UTF-8"); for (String k : params.keySet()) { for (Object v : params.getValues(k)) { entry.getRequest().getQueryString().add(new HarNameValuePair(k, (String) v)); } } } String errorMessage = null; HttpResponse response = null; BasicHttpContext ctx = new BasicHttpContext(); ActiveRequest activeRequest = new ActiveRequest(method, ctx, entry.getStartedDateTime()); synchronized (activeRequests) { activeRequests.add(activeRequest); } // for dealing with automatic authentication if (authType == AuthType.NTLM) { // todo: not supported yet //ctx.setAttribute("preemptive-auth", new NTLMScheme(new JCIFSEngine())); } else if (authType == AuthType.BASIC) { ctx.setAttribute("preemptive-auth", new BasicScheme()); } StatusLine statusLine = null; try { // set the User-Agent if it's not already set if (method.getHeaders("User-Agent").length == 0) { method.addHeader("User-Agent", "BrowserMob VU/1.0"); } // was the request mocked out? if (mockResponseCode != -1) { statusCode = mockResponseCode; // TODO: HACKY!! callback.handleHeaders(new Header[] { new Header() { @Override public String getName() { return "Content-Type"; } @Override public String getValue() { return "text/plain"; } @Override public HeaderElement[] getElements() throws ParseException { return new HeaderElement[0]; } } }); // Make sure we set the status line here too. // Use the version number from the request ProtocolVersion version = null; int reqDotVersion = req.getProxyRequest().getDotVersion(); if (reqDotVersion == -1) { version = new HttpVersion(0, 9); } else if (reqDotVersion == 0) { version = new HttpVersion(1, 0); } else if (reqDotVersion == 1) { version = new HttpVersion(1, 1); } // and if not any of these, trust that a Null version will // cause an appropriate error callback.handleStatusLine( new BasicStatusLine(version, statusCode, "Status set by browsermob-proxy")); // No mechanism to look up the response text by status code, // so include a notification that this is a synthetic error code. } else { response = httpClient.execute(method, ctx); statusLine = response.getStatusLine(); statusCode = statusLine.getStatusCode(); if (callback != null) { callback.handleStatusLine(statusLine); callback.handleHeaders(response.getAllHeaders()); } if (response.getEntity() != null) { is = response.getEntity().getContent(); } // check for null (resp 204 can cause HttpClient to return null, which is what Google does with http://clients1.google.com/generate_204) if (is != null) { Header contentEncodingHeader = response.getFirstHeader("Content-Encoding"); if (contentEncodingHeader != null && "gzip".equalsIgnoreCase(contentEncodingHeader.getValue())) { gzipping = true; } // deal with GZIP content! if (decompress && gzipping) { is = new GZIPInputStream(is); } if (captureContent) { // todo - something here? os = new ClonedOutputStream(os); } bytes = copyWithStats(is, os); } } } catch (Exception e) { errorMessage = e.toString(); if (callback != null) { callback.reportError(e); } // only log it if we're not shutdown (otherwise, errors that happen during a shutdown can likely be ignored) if (!shutdown) { LOG.info(String.format("%s when requesting %s", errorMessage, url)); } } finally { // the request is done, get it out of here synchronized (activeRequests) { activeRequests.remove(activeRequest); } if (is != null) { try { is.close(); } catch (IOException e) { // this is OK to ignore } } } // record the response as ended RequestInfo.get().finish(); // set the start time and other timings entry.setStartedDateTime(RequestInfo.get().getStart()); entry.setTimings(RequestInfo.get().getTimings()); entry.setServerIPAddress(RequestInfo.get().getResolvedAddress()); entry.setTime(RequestInfo.get().getTotalTime()); // todo: where you store this in HAR? // obj.setErrorMessage(errorMessage); entry.getResponse().setBodySize(bytes); entry.getResponse().getContent().setSize(bytes); entry.getResponse().setStatus(statusCode); if (statusLine != null) { entry.getResponse().setStatusText(statusLine.getReasonPhrase()); } boolean urlEncoded = false; if (captureHeaders || captureContent) { for (Header header : method.getAllHeaders()) { if (header.getValue() != null && header.getValue().startsWith(URLEncodedUtils.CONTENT_TYPE)) { urlEncoded = true; } entry.getRequest().getHeaders().add(new HarNameValuePair(header.getName(), header.getValue())); } if (response != null) { for (Header header : response.getAllHeaders()) { entry.getResponse().getHeaders().add(new HarNameValuePair(header.getName(), header.getValue())); } } } if (captureContent) { // can we understand the POST data at all? if (method instanceof HttpEntityEnclosingRequestBase && req.getCopy() != null) { HttpEntityEnclosingRequestBase enclosingReq = (HttpEntityEnclosingRequestBase) method; HttpEntity entity = enclosingReq.getEntity(); HarPostData data = new HarPostData(); data.setMimeType(req.getMethod().getFirstHeader("Content-Type").getValue()); entry.getRequest().setPostData(data); if (urlEncoded || URLEncodedUtils.isEncoded(entity)) { try { final String content = new String(req.getCopy().toByteArray(), "UTF-8"); if (content != null && content.length() > 0) { List<NameValuePair> result = new ArrayList<NameValuePair>(); URLEncodedUtils.parse(result, new Scanner(content), null); ArrayList<HarPostDataParam> params = new ArrayList<HarPostDataParam>(); data.setParams(params); for (NameValuePair pair : result) { params.add(new HarPostDataParam(pair.getName(), pair.getValue())); } } } catch (Exception e) { LOG.info("Unexpected problem when parsing input copy", e); } } else { // not URL encoded, so let's grab the body of the POST and capture that data.setText(new String(req.getCopy().toByteArray())); } } } //capture request cookies javax.servlet.http.Cookie[] cookies = req.getProxyRequest().getCookies(); for (javax.servlet.http.Cookie cookie : cookies) { HarCookie hc = new HarCookie(); hc.setName(cookie.getName()); hc.setValue(cookie.getValue()); entry.getRequest().getCookies().add(hc); } String contentType = null; if (response != null) { try { Header contentTypeHdr = response.getFirstHeader("Content-Type"); if (contentTypeHdr != null) { contentType = contentTypeHdr.getValue(); entry.getResponse().getContent().setMimeType(contentType); if (captureContent && os != null && os instanceof ClonedOutputStream) { ByteArrayOutputStream copy = ((ClonedOutputStream) os).getOutput(); if (gzipping) { // ok, we need to decompress it before we can put it in the har file try { InputStream temp = new GZIPInputStream( new ByteArrayInputStream(copy.toByteArray())); copy = new ByteArrayOutputStream(); IOUtils.copy(temp, copy); } catch (IOException e) { throw new RuntimeException(e); } } if (hasTextualContent(contentType)) { setTextOfEntry(entry, copy, contentType); } else if (captureBinaryContent) { setBinaryContentOfEntry(entry, copy); } } NameValuePair nvp = contentTypeHdr.getElements()[0].getParameterByName("charset"); if (nvp != null) { charSet = nvp.getValue(); } } if (os instanceof ByteArrayOutputStream) { responseBody = ((ByteArrayOutputStream) os).toString(charSet); if (verificationText != null) { contentMatched = responseBody.contains(verificationText); } } } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } if (contentType != null) { entry.getResponse().getContent().setMimeType(contentType); } // checking to see if the client is being redirected boolean isRedirect = false; String location = null; if (response != null && statusCode >= 300 && statusCode < 400 && statusCode != 304) { isRedirect = true; // pulling the header for the redirect Header locationHeader = response.getLastHeader("location"); if (locationHeader != null) { location = locationHeader.getValue(); } else if (this.followRedirects) { throw new RuntimeException("Invalid redirect - missing location header"); } } // // Response validation - they only work if we're not following redirects // int expectedStatusCode = req.getExpectedStatusCode(); // if we didn't mock out the actual response code and the expected code isn't what we saw, we have a problem if (mockResponseCode == -1 && expectedStatusCode > -1) { if (this.followRedirects) { throw new RuntimeException("Response validation cannot be used while following redirects"); } if (expectedStatusCode != statusCode) { if (isRedirect) { throw new RuntimeException("Expected status code of " + expectedStatusCode + " but saw " + statusCode + " redirecting to: " + location); } else { throw new RuntimeException( "Expected status code of " + expectedStatusCode + " but saw " + statusCode); } } } // Location header check: if (isRedirect && (req.getExpectedLocation() != null)) { if (this.followRedirects) { throw new RuntimeException("Response validation cannot be used while following redirects"); } if (location.compareTo(req.getExpectedLocation()) != 0) { throw new RuntimeException( "Expected a redirect to " + req.getExpectedLocation() + " but saw " + location); } } // end of validation logic // basic tail recursion for redirect handling if (isRedirect && this.followRedirects) { // updating location: try { URI redirectUri = new URI(location); URI newUri = method.getURI().resolve(redirectUri); method.setURI(newUri); return execute(req, ++depth); } catch (URISyntaxException e) { LOG.warn("Could not parse URL", e); } } return new BrowserMobHttpResponse(entry, method, response, contentMatched, verificationText, errorMessage, responseBody, contentType, charSet); }