List of usage examples for org.apache.commons.httpclient HttpMethodBase getResponseContentLength
public long getResponseContentLength()
From source file:com.esri.gpt.framework.http.HttpClientRequest.java
/** * Determines basic information associted with an HTTP response. * <br/>For example: response status message, Content-Type, Content-Length * @param method the HttpMethod that was executed *///from w ww .j av a2s . c o m private void determineResponseInfo(HttpMethodBase method) { this.getResponseInfo().setResponseMessage(method.getStatusText()); this.getResponseInfo().setResponseHeaders(method.getResponseHeaders()); Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { HeaderElement values[] = contentTypeHeader.getElements(); // Expect only one header element to be there, no more, no less if (values.length == 1) { this.getResponseInfo().setContentType(values[0].getName()); NameValuePair param = values[0].getParameterByName("charset"); if (param != null) { // If invalid, an UnsupportedEncondingException will result this.getResponseInfo().setContentEncoding(param.getValue()); } } } this.getResponseInfo().setContentLength(method.getResponseContentLength()); }
From source file:com.atlantbh.jmeter.plugins.oauth.OAuthSampler.java
@Override public SampleResult sample() { HttpMethodBase httpMethod = null; HttpClient client = null;// w w w.ja va 2s. c o m InputStream instream = null; SampleResult res = new SampleResult(); try { res.setSuccessful(false); res.setResponseCode("000"); res.setSampleLabel(getName()); res.setURL(getUrl()); res.setDataEncoding("UTF-8"); res.setDataType("text/xml"); res.setSamplerData(getRequestBody()); res.setMonitor(isMonitor()); res.sampleStart(); String urlStr = getUrl().toString(); log.debug("Start : sample " + urlStr); log.debug("method " + getMethod()); httpMethod = createHttpMethod(getMethod(), urlStr); setDefaultRequestHeaders(httpMethod); client = setupConnection(getUrl(), httpMethod); if (httpMethod instanceof EntityEnclosingMethod) { ((EntityEnclosingMethod) httpMethod) .setRequestEntity(new StringRequestEntity(getRequestBody(), "text/xml", "UTF-8")); } overrideHeaders(httpMethod, urlStr, getMethod()); res.setRequestHeaders(getConnectionHeaders(httpMethod)); int statusCode = -1; try { statusCode = client.executeMethod(httpMethod); } catch (RuntimeException e) { log.error("Exception when executing '" + httpMethod + "'", e); throw e; } instream = httpMethod.getResponseBodyAsStream(); if (instream != null) { Header responseHeader = httpMethod.getResponseHeader(HEADER_CONTENT_ENCODING); if (responseHeader != null && ENCODING_GZIP.equals(responseHeader.getValue())) { instream = new GZIPInputStream(instream); } res.setResponseData(readResponse(res, instream, (int) httpMethod.getResponseContentLength())); } res.sampleEnd(); res.setResponseCode(Integer.toString(statusCode)); res.setSuccessful(isSuccessCode(statusCode)); res.setResponseMessage(httpMethod.getStatusText()); String ct = null; org.apache.commons.httpclient.Header h = httpMethod.getResponseHeader(HEADER_CONTENT_TYPE); if (h != null) { ct = h.getValue(); res.setContentType(ct); res.setEncodingAndType(ct); } String responseHeaders = getResponseHeaders(httpMethod); res.setResponseHeaders(responseHeaders); log.debug("End : sample"); httpMethod.releaseConnection(); return res; } catch (MalformedURLException e) { res.sampleEnd(); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } catch (IllegalArgumentException e) { res.sampleEnd(); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } catch (IOException e) { res.sampleEnd(); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } finally { JOrphanUtils.closeQuietly(instream); if (httpMethod != null) { httpMethod.releaseConnection(); return res; } } }
From source file:com.snaker.DownloadManager.java
private Runnable createRunnable(final Downloader d) { final Task task = d.getTask(); return new Runnable() { @Override/*from w w w. j ava2 s.co m*/ public void run() { logger.info("start download:" + d.getUrl()); HttpClient client = clients.get(task.getId()); DownloadHandler handler = d.getHandler(); HttpMethodBase m = null; d.setStatus(Status.STARTED); d.setStartTime(System.currentTimeMillis()); try { String url = d.getUrl(); if (d.isGet()) { GetMethod get = new GetMethod(url); m = get; } else { final String requestCharset = d.getRequestCharset(); PostMethod post = new PostMethod(url) { public String getRequestCharSet() { if (requestCharset != null) return requestCharset; else return super.getRequestCharSet(); } public boolean getFollowRedirects() { return d.isFollowRedirects(); } }; if (requestCharset != null) { post.setRequestHeader("ContentType", "application/x-www-form-urlencoded;charset=" + requestCharset); } DownloadParams parms = d.getParms(); if (parms != null) post.setRequestBody(parms.toNVP()); m = post; } { // set the headers m.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"); m.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); m.setRequestHeader("Accept-Language", "en-us,zh-cn;q=0.5"); m.setRequestHeader("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); m.setRequestHeader("Referer", url); } client.executeMethod(m); //check status int sc = m.getStatusCode(); d.setStatusCode(sc); if (isBadStatusCode(sc)) { logger.error("download failed,url:" + d.getUrl() + ",Status Code:" + sc); d.setStatus(Status.FAILED); d.setDescription(m.getStatusText()); return; } else if (sc == 404 || sc == 410) { d.setStatus(Status.FINISHED); d.setDescription("NOT FOUND"); return; } long size = m.getResponseContentLength(); d.setFileSize(size); // get File Name if (d.getFileName() == null) { Header h = m.getResponseHeader("Content-Disposition"); String fileName = null; if (h != null) { String f = h.getValue(); int tag = f.indexOf("filename="); if (tag != -1 && tag != f.length() - 1) fileName = f.substring(tag + 1); } if (fileName == null || fileName.length() == 0) { int tag1 = url.lastIndexOf("/"); int tag2 = url.lastIndexOf("?"); if (tag1 != -1 && tag1 != url.length() - 1) { if (tag2 > tag1) { fileName = url.substring(tag1 + 1, tag2); } else { fileName = url.substring(tag1 + 1); } } } d.setFileName(fileName); } // set the all headers Header[] headers = m.getResponseHeaders(); if (headers != null) { for (Header header : headers) { d.addResponseHeader(header.getName(), header.getValue()); } } d.setStatus(Status.RUNNING); // recv the body if (handler == null) { byte[] content = m.getResponseBody(); int len = content.length; d.setFileSize(len); d.setReceived(len); d.setResponseCharset(m.getResponseCharSet()); d.setResponseBody(content); } else { InputStream is = m.getResponseBodyAsStream(); handler.start(d); byte[] buffer = new byte[102400]; long count = 0; while (true) { int r = is.read(buffer); if (r > 0) { count += r; d.setReceived(count); handler.handle(buffer, r); } else { break; } } is.close(); } d.setStatus(Status.FINISHED); } catch (Exception e) { logger.error("download failed,url:" + d.getUrl(), e); d.setStatus(Status.FAILED); d.setDescription(e.getMessage()); } finally { m.releaseConnection(); if (handler != null) { handler.stop(); } downloadings.remove(d); d.setEndTime(System.currentTimeMillis()); while (downloaded.size() >= maxDownloadedCount) { downloaded.poll(); } downloaded.offer(d); task.downloadFininshed(d); } } }; }
From source file:com.atlantbh.jmeter.plugins.rest.RestSampler.java
/** * Method invoked by JMeter when a sample needs to happen. It's actually an * indirect call from the main sampler interface. it's resolved in the base * class.//from w w w. ja v a 2s . c o m * * This is a copy and paste from the HTTPSampler2 - quick and dirty hack as * that class is not very extensible. The reason to extend and slightly * modify is that I needed to get the body content from a text field in the * GUI rather than a file. */ public SampleResult sample() { HttpMethodBase httpMethod = null; HttpClient client = null; InputStream instream = null; SampleResult res = new SampleResult(); try { res.setSuccessful(false); res.setResponseCode("000"); res.setSampleLabel(getName()); res.setURL(getUrl()); res.setDataEncoding("UTF-8"); res.setDataType("text/xml"); res.setMonitor(isMonitor()); res.sampleStart(); String urlStr = getUrl().toString(); String request = getMethod().toString() + " " + urlStr + "\n"; request += getRequestBody().toString(); res.setSamplerData(request); log.debug("Start : sample " + urlStr); log.debug("method " + getMethod()); httpMethod = createHttpMethod(getMethod(), urlStr); // Set any default request headers setDefaultRequestHeaders(httpMethod); // Setup connection client = setupConnection(getUrl(), httpMethod); // Handle the various methods if (httpMethod instanceof EntityEnclosingMethod) { ((EntityEnclosingMethod) httpMethod) .setRequestEntity(new StringRequestEntity(getRequestBody(), "text/xml", "UTF-8")); //res.setQueryString(getRequestBody()); //String postBody = sendData((EntityEnclosingMethod) httpMethod); //res.setResponseData(postBody.getBytes()); //String postBody = ""; //try { postBody = Base64Util.processStargateRequest(getRequestBody());} //catch (Exception e) {postBody =getRequestBody(); e.printStackTrace();} // //res.setSamplerData(/*res.getSamplerData() + "\r\n*/"ORIGINAL CONTENT:\r\n\r\n" + getRequestBody() + "\r\n\r\nSENT CONTENT:\r\n\r\n" + postBody); } overrideHeaders(httpMethod); res.setRequestHeaders(getConnectionHeaders(httpMethod)); int statusCode = -1; try { statusCode = client.executeMethod(httpMethod); } catch (RuntimeException e) { log.error("Exception when executing '" + httpMethod + "'", e); throw e; } // Request sent. Now get the response: instream = httpMethod.getResponseBodyAsStream(); if (instream != null) {// will be null for HEAD Header responseHeader = httpMethod.getResponseHeader(HEADER_CONTENT_ENCODING); if (responseHeader != null && ENCODING_GZIP.equals(responseHeader.getValue())) { instream = new GZIPInputStream(instream); } res.setResponseData(readResponse(res, instream, (int) httpMethod.getResponseContentLength())); } //if (res.getResponseDataAsString().length() != 0) // try{ // String response = Base64Util.processStargateResponse(res.getResponseDataAsString()); // res.setResponseData(response); // } catch (Exception e) // { // log.error(e.getMessage()); // } res.sampleEnd(); // Done with the sampling proper. // Now collect the results into the HTTPSampleResult: // res.setSampleLabel(httpMethod.getURI().toString()); // Pick up Actual path (after redirects) res.setResponseCode(Integer.toString(statusCode)); res.setSuccessful(isSuccessCode(statusCode)); res.setResponseMessage(httpMethod.getStatusText()); String ct = null; org.apache.commons.httpclient.Header h = httpMethod.getResponseHeader(HEADER_CONTENT_TYPE); if (h != null)// Can be missing, e.g. on redirect { ct = h.getValue(); res.setContentType(ct);// e.g. text/html; charset=ISO-8859-1 res.setEncodingAndType(ct); } String responseHeaders = getResponseHeaders(httpMethod); res.setResponseHeaders(responseHeaders); /*if (res.isRedirect()) { final Header headerLocation = httpMethod.getResponseHeader(HEADER_LOCATION); if (headerLocation == null) { // HTTP protocol violation, but // avoids NPE throw new IllegalArgumentException("Missing location header"); } res.setRedirectLocation(headerLocation.getValue()); } // If we redirected automatically, the URL may have changed if (getAutoRedirects()) { res.setURL(new URL(httpMethod.getURI().toString())); } // Store any cookies received in the cookie manager: saveConnectionCookies(httpMethod, res.getURL(), getCookieManager()); // Save cache information final CacheManager cacheManager = getCacheManager(); if (cacheManager != null) { cacheManager.saveDetails(httpMethod, res); } // Follow redirects and download page resources if appropriate: res = resultProcessing(areFollowingRedirect, frameDepth, res); */ log.debug("End : sample"); httpMethod.releaseConnection(); return res; } catch (MalformedURLException e) { res.sampleEnd(); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } catch (IllegalArgumentException e)// e.g. some kinds of invalid URL { res.sampleEnd(); //SampleResult err = errorResult(e, res); //err.setSampleLabel("Error: " + url.toString()); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } catch (IOException e) { res.sampleEnd(); log.warn(e.getMessage()); res.setResponseMessage(e.getMessage()); return res; } finally { JOrphanUtils.closeQuietly(instream); if (httpMethod != null) { httpMethod.releaseConnection(); return res; } } }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
protected void innerProcess(final CrawlURI curi) throws InterruptedException { if (!canFetch(curi)) { // Cannot fetch this, due to protocol, retries, or other problems return;/*from w w w . j av a 2 s.c om*/ } this.curisHandled++; // Note begin time curi.putLong(A_FETCH_BEGAN_TIME, System.currentTimeMillis()); // Get a reference to the HttpRecorder that is set into this ToeThread. HttpRecorder rec = HttpRecorder.getHttpRecorder(); // Shall we get a digest on the content downloaded? boolean digestContent = ((Boolean) getUncheckedAttribute(curi, ATTR_DIGEST_CONTENT)).booleanValue(); String algorithm = null; if (digestContent) { algorithm = ((String) getUncheckedAttribute(curi, ATTR_DIGEST_ALGORITHM)); rec.getRecordedInput().setDigest(algorithm); } else { // clear rec.getRecordedInput().setDigest((MessageDigest) null); } // Below we do two inner classes that add check of midfetch // filters just as we're about to receive the response body. String curiString = curi.getUURI().toString(); HttpMethodBase method = null; if (curi.isPost()) { method = new HttpRecorderPostMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } else { method = new HttpRecorderGetMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } HostConfiguration customConfigOrNull = configureMethod(curi, method); // Set httpRecorder into curi. Subsequent code both here and later // in extractors expects to find the HttpRecorder in the CrawlURI. curi.setHttpRecorder(rec); // Populate credentials. Set config so auth. is not automatic. boolean addedCredentials = populateCredentials(curi, method); method.setDoAuthentication(addedCredentials); // set hardMax on bytes (if set by operator) long hardMax = getMaxLength(curi); // set overall timeout (if set by operator) long timeoutMs = 1000 * getTimeout(curi); // Get max fetch rate (bytes/ms). It comes in in KB/sec long maxRateKBps = getMaxFetchRate(curi); rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps); try { this.http.executeMethod(customConfigOrNull, method); } catch (RecorderTooMuchHeaderException ex) { // when too much header material, abort like other truncations doAbort(curi, method, HEADER_TRUNC); } catch (IOException e) { failedExecuteCleanup(method, curi, e); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions in native // code... see // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException failedExecuteCleanup(method, curi, e); return; } // set softMax on bytes to get (if implied by content-length) long softMax = method.getResponseContentLength(); try { if (!method.isAborted()) { // Force read-to-end, so that any socket hangs occur here, // not in later modules. rec.getRecordedInput().readFullyOrUntil(softMax); } } catch (RecorderTimeoutException ex) { doAbort(curi, method, TIMER_TRUNC); } catch (RecorderLengthExceededException ex) { doAbort(curi, method, LENGTH_TRUNC); } catch (IOException e) { cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions from native code // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } finally { // ensure recording has stopped rec.closeRecorders(); if (!method.isAborted()) { method.releaseConnection(); } // Note completion time curi.putLong(A_FETCH_COMPLETED_TIME, System.currentTimeMillis()); // Set the response charset into the HttpRecord if available. setCharacterEncoding(rec, method); setSizes(curi, rec); } if (digestContent) { curi.setContentDigest(algorithm, rec.getRecordedInput().getDigestValue()); } if (logger.isLoggable(Level.INFO)) { logger.info((curi.isPost() ? "POST" : "GET") + " " + curi.getUURI().toString() + " " + method.getStatusCode() + " " + rec.getRecordedInput().getSize() + " " + curi.getContentType()); } if (curi.isSuccess() && addedCredentials) { // Promote the credentials from the CrawlURI to the CrawlServer // so they are available for all subsequent CrawlURIs on this // server. promoteCredentials(curi); if (logger.isLoggable(Level.FINE)) { // Print out the cookie. Might help with the debugging. Header setCookie = method.getResponseHeader("set-cookie"); if (setCookie != null) { logger.fine(setCookie.toString().trim()); } } } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { // 401 is not 'success'. handle401(method, curi); } if (rec.getRecordedInput().isOpen()) { logger.severe(curi.toString() + " RIS still open. Should have" + " been closed by method release: " + Thread.currentThread().getName()); try { rec.getRecordedInput().close(); } catch (IOException e) { logger.log(Level.SEVERE, "second-chance RIS close failed", e); } } }
From source file:com.sittinglittleduck.DirBuster.Worker.java
/** Run method of the thread */ public void run() { queue = manager.workQueue;//from www. j av a2s . c om while (manager.hasWorkLeft()) { working = false; // code to make the worker pause, if the pause button has been presed // if the stop signal has been given stop the thread if (stop) { return; } // this pauses the thread synchronized (this) { while (pleaseWait) { try { wait(); } catch (InterruptedException e) { return; } catch (Exception e) { e.printStackTrace(); } } } HttpMethodBase httpMethod = null; try { work = (WorkUnit) queue.take(); working = true; url = work.getWork(); int code = 0; String response = ""; String rawResponse = ""; httpMethod = createHttpMethod(work.getMethod(), url.toString()); // if the work is a head request if (work.getMethod().equalsIgnoreCase("HEAD")) { code = makeRequest(httpMethod); httpMethod.releaseConnection(); } // if we are doing a get request else if (work.getMethod().equalsIgnoreCase("GET")) { code = makeRequest(httpMethod); String rawHeader = getHeadersAsString(httpMethod); response = getResponseAsString(httpMethod); rawResponse = rawHeader + response; // clean the response if (Config.parseHTML && !work.getBaseCaseObj().isUseRegexInstead()) { parseHtml(httpMethod, response); } response = FilterResponce.CleanResponce(response, work); Thread.sleep(10); httpMethod.releaseConnection(); } // if we need to check the against the base case if (work.getMethod().equalsIgnoreCase("GET") && work.getBaseCaseObj().useContentAnalysisMode()) { if (code == HttpStatus.SC_OK) { verifyResponseForValidRequests(code, response, rawResponse); } else if (code == HttpStatus.SC_NOT_FOUND || code == HttpStatus.SC_BAD_REQUEST) { if (Config.debug) { System.out .println("DEBUG Worker[" + threadId + "]: " + code + " for: " + url.toString()); } } else { notifyItemFound(code, response, rawResponse, work.getBaseCaseObj().getBaseCase()); } } /* * use the custom regex check instead */ else if (work.getBaseCaseObj().isUseRegexInstead()) { Pattern regexFindFile = Pattern.compile(work.getBaseCaseObj().getRegex()); Matcher m = regexFindFile.matcher(rawResponse); if (m.find()) { // do nothing as we have a 404 if (Config.debug) { System.out.println("DEBUG Worker[" + threadId + "]: Regex matched so it's a 404, " + url.toString()); } } else { if (Config.parseHTML) { parseHtml(httpMethod, rawResponse); } notifyItemFound(code, response, rawResponse, work.getBaseCaseObj().getBaseCase()); } } // just check the response code else { // if is not the fail code, a 404 or a 400 then we have a possible if (code != work.getBaseCaseObj().getFailCode() && verifyIfCodeIsValid(code)) { if (work.getMethod().equalsIgnoreCase("HEAD")) { httpMethod = createHttpMethod("GET", url.toString()); int newCode = makeRequest(httpMethod); // in some cases the second get can return a different result, than the // first head request! if (newCode != code) { manager.foundError(url, "Return code for first HEAD, is different to the second GET: " + code + " - " + newCode); } // build a string version of the headers rawResponse = getHeadersAsString(httpMethod); if (httpMethod.getResponseContentLength() > 0) { String responseBodyAsString = getResponseAsString(httpMethod); rawResponse = rawResponse + responseBodyAsString; if (Config.parseHTML) { parseHtml(httpMethod, responseBodyAsString); } } httpMethod.releaseConnection(); } if (work.isDir()) { manager.foundDir(url, code, rawResponse, work.getBaseCaseObj()); } else { manager.foundFile(url, code, rawResponse, work.getBaseCaseObj()); } } } manager.workDone(); Thread.sleep(20); } catch (NoHttpResponseException e) { manager.foundError(url, "NoHttpResponseException " + e.getMessage()); manager.workDone(); } catch (ConnectTimeoutException e) { manager.foundError(url, "ConnectTimeoutException " + e.getMessage()); manager.workDone(); } catch (URIException e) { manager.foundError(url, "URIException " + e.getMessage()); manager.workDone(); } catch (IOException e) { manager.foundError(url, "IOException " + e.getMessage()); manager.workDone(); } catch (InterruptedException e) { // manager.foundError(url, "InterruptedException " + e.getMessage()); manager.workDone(); return; } catch (IllegalArgumentException e) { e.printStackTrace(); manager.foundError(url, "IllegalArgumentException " + e.getMessage()); manager.workDone(); } finally { if (httpMethod != null) { httpMethod.releaseConnection(); } } } }
From source file:org.alfresco.repo.remoteconnector.RemoteConnectorServiceImpl.java
/** * Executes the specified request, and return the response *///ww w. j av a2s . co m public RemoteConnectorResponse executeRequest(RemoteConnectorRequest request) throws IOException, AuthenticationException, RemoteConnectorClientException, RemoteConnectorServerException { RemoteConnectorRequestImpl reqImpl = (RemoteConnectorRequestImpl) request; HttpMethodBase httpRequest = reqImpl.getMethodInstance(); // Attach the headers to the request for (Header hdr : request.getRequestHeaders()) { httpRequest.addRequestHeader(hdr); } // Attach the body, if possible if (httpRequest instanceof EntityEnclosingMethod) { if (request.getRequestBody() != null) { ((EntityEnclosingMethod) httpRequest).setRequestEntity(reqImpl.getRequestBody()); } } // Grab our thread local HttpClient instance // Remember - we must then clean it up! HttpClient httpClient = HttpClientHelper.getHttpClient(); // The url should already be vetted by the RemoteConnectorRequest URL url = new URL(request.getURL()); // Use the appropriate Proxy Host if required if (httpProxyHost != null && url.getProtocol().equals("http") && requiresProxy(url.getHost())) { httpClient.getHostConfiguration().setProxyHost(httpProxyHost); if (logger.isDebugEnabled()) logger.debug(" - using HTTP proxy host for: " + url); if (httpProxyCredentials != null) { httpClient.getState().setProxyCredentials(httpAuthScope, httpProxyCredentials); if (logger.isDebugEnabled()) logger.debug(" - using HTTP proxy credentials for proxy: " + httpProxyHost.getHostName()); } } else if (httpsProxyHost != null && url.getProtocol().equals("https") && requiresProxy(url.getHost())) { httpClient.getHostConfiguration().setProxyHost(httpsProxyHost); if (logger.isDebugEnabled()) logger.debug(" - using HTTPS proxy host for: " + url); if (httpsProxyCredentials != null) { httpClient.getState().setProxyCredentials(httpsAuthScope, httpsProxyCredentials); if (logger.isDebugEnabled()) logger.debug(" - using HTTPS proxy credentials for proxy: " + httpsProxyHost.getHostName()); } } else { //host should not be proxied remove any configured proxies httpClient.getHostConfiguration().setProxyHost(null); httpClient.getState().clearProxyCredentials(); } // Log what we're doing if (logger.isDebugEnabled()) { logger.debug("Performing " + request.getMethod() + " request to " + request.getURL()); for (Header hdr : request.getRequestHeaders()) { logger.debug("Header: " + hdr); } Object requestBody = null; if (request != null) { requestBody = request.getRequestBody(); } if (requestBody != null && requestBody instanceof StringRequestEntity) { StringRequestEntity re = (StringRequestEntity) request.getRequestBody(); logger.debug("Payload (string): " + re.getContent()); } else if (requestBody != null && requestBody instanceof ByteArrayRequestEntity) { ByteArrayRequestEntity re = (ByteArrayRequestEntity) request.getRequestBody(); logger.debug("Payload (byte array): " + re.getContent().toString()); } else { logger.debug("Payload is not of a readable type."); } } // Perform the request, and wrap the response int status = -1; String statusText = null; RemoteConnectorResponse response = null; try { status = httpClient.executeMethod(httpRequest); statusText = httpRequest.getStatusText(); Header[] responseHdrs = httpRequest.getResponseHeaders(); Header responseContentTypeH = httpRequest .getResponseHeader(RemoteConnectorRequestImpl.HEADER_CONTENT_TYPE); String responseCharSet = httpRequest.getResponseCharSet(); String responseContentType = (responseContentTypeH != null ? responseContentTypeH.getValue() : null); if (logger.isDebugEnabled()) { logger.debug( "response url=" + request.getURL() + ", length =" + httpRequest.getResponseContentLength() + ", responceContentType " + responseContentType + ", statusText =" + statusText); } // Decide on how best to handle the response, based on the size // Ideally, we want to close the HttpClient resources immediately, but // that isn't possible for very large responses // If we can close immediately, it makes cleanup simpler and fool-proof if (httpRequest.getResponseContentLength() > MAX_BUFFER_RESPONSE_SIZE || httpRequest.getResponseContentLength() == -1) { if (logger.isTraceEnabled()) { logger.trace("large response (or don't know length) url=" + request.getURL()); } // Need to wrap the InputStream in something that'll close InputStream wrappedStream = new HttpClientReleasingInputStream(httpRequest); httpRequest = null; // Now build the response response = new RemoteConnectorResponseImpl(request, responseContentType, responseCharSet, status, responseHdrs, wrappedStream); } else { if (logger.isTraceEnabled()) { logger.debug("small response for url=" + request.getURL()); } // Fairly small response, just keep the bytes and make life simple response = new RemoteConnectorResponseImpl(request, responseContentType, responseCharSet, status, responseHdrs, httpRequest.getResponseBody()); // Now we have the bytes, we can close the HttpClient resources httpRequest.releaseConnection(); httpRequest = null; } } finally { // Make sure, problems or not, we always tidy up (if not large stream based) // This is important because we use a thread local HttpClient instance if (httpRequest != null) { httpRequest.releaseConnection(); httpRequest = null; } } // Log the response if (logger.isDebugEnabled()) logger.debug("Response was " + status + " " + statusText); // Decide if we should throw an exception if (status >= 300) { // Tidy if needed if (httpRequest != null) httpRequest.releaseConnection(); // Specific exceptions if (status == Status.STATUS_FORBIDDEN || status == Status.STATUS_UNAUTHORIZED) { // TODO Forbidden may need to be handled differently. // TODO Need to get error message into the AuthenticationException throw new AuthenticationException(statusText); } // Server side exceptions if (status >= 500 && status <= 599) { logger.error("executeRequest: remote connector server exception: [" + status + "] " + statusText); throw new RemoteConnectorServerException(status, statusText); } if (status == Status.STATUS_PRECONDITION_FAILED) { logger.error("executeRequest: remote connector client exception: [" + status + "] " + statusText); throw new RemoteConnectorClientException(status, statusText, response); } else { // Client request exceptions if (httpRequest != null) { // Response wasn't too big and is available, supply it logger.error( "executeRequest: remote connector client exception: [" + status + "] " + statusText); throw new RemoteConnectorClientException(status, statusText, response); } else { // Response was too large, report without it logger.error( "executeRequest: remote connector client exception: [" + status + "] " + statusText); throw new RemoteConnectorClientException(status, statusText, null); } } } // If we get here, then the request/response was all fine // So, return our created response return response; }
From source file:org.apache.jmeter.protocol.http.sampler.HTTPHC3Impl.java
/** * Samples the URL passed in and stores the result in * <code>HTTPSampleResult</code>, following redirects and downloading * page resources as appropriate.//from w ww. j av a2 s . com * <p> * When getting a redirect target, redirects are not followed and resources * are not downloaded. The caller will take care of this. * * @param url * URL to sample * @param method * HTTP method: GET, POST,... * @param areFollowingRedirect * whether we're getting a redirect target * @param frameDepth * Depth of this target in the frame structure. Used only to * prevent infinite recursion. * @return results of the sampling */ @Override protected HTTPSampleResult sample(URL url, String method, boolean areFollowingRedirect, int frameDepth) { String urlStr = url.toString(); if (log.isDebugEnabled()) { log.debug("Start : sample " + urlStr); log.debug("method " + method + " followingRedirect " + areFollowingRedirect + " depth " + frameDepth); } HttpMethodBase httpMethod = null; HTTPSampleResult res = new HTTPSampleResult(); res.setMonitor(isMonitor()); res.setSampleLabel(urlStr); // May be replaced later res.setHTTPMethod(method); res.setURL(url); res.sampleStart(); // Count the retries as well in the time try { // May generate IllegalArgumentException if (method.equals(HTTPConstants.POST)) { httpMethod = new PostMethod(urlStr); } else if (method.equals(HTTPConstants.GET)) { httpMethod = new GetMethod(urlStr); } else if (method.equals(HTTPConstants.PUT)) { httpMethod = new PutMethod(urlStr); } else if (method.equals(HTTPConstants.HEAD)) { httpMethod = new HeadMethod(urlStr); } else if (method.equals(HTTPConstants.TRACE)) { httpMethod = new TraceMethod(urlStr); } else if (method.equals(HTTPConstants.OPTIONS)) { httpMethod = new OptionsMethod(urlStr); } else if (method.equals(HTTPConstants.DELETE)) { httpMethod = new EntityEnclosingMethod(urlStr) { @Override public String getName() { // HC3.1 does not have the method return HTTPConstants.DELETE; } }; } else if (method.equals(HTTPConstants.PATCH)) { httpMethod = new EntityEnclosingMethod(urlStr) { @Override public String getName() { // HC3.1 does not have the method return HTTPConstants.PATCH; } }; } else { throw new IllegalArgumentException("Unexpected method: '" + method + "'"); } final CacheManager cacheManager = getCacheManager(); if (cacheManager != null && HTTPConstants.GET.equalsIgnoreCase(method)) { if (cacheManager.inCache(url)) { return updateSampleResultForResourceInCache(res); } } // Set any default request headers setDefaultRequestHeaders(httpMethod); // Setup connection HttpClient client = setupConnection(url, httpMethod, res); savedClient = client; // Handle the various methods if (method.equals(HTTPConstants.POST)) { String postBody = sendPostData((PostMethod) httpMethod); res.setQueryString(postBody); } else if (method.equals(HTTPConstants.PUT) || method.equals(HTTPConstants.PATCH) || method.equals(HTTPConstants.DELETE)) { String putBody = sendEntityData((EntityEnclosingMethod) httpMethod); res.setQueryString(putBody); } int statusCode = client.executeMethod(httpMethod); // We've finished with the request, so we can add the LocalAddress to it for display final InetAddress localAddr = client.getHostConfiguration().getLocalAddress(); if (localAddr != null) { httpMethod.addRequestHeader(HEADER_LOCAL_ADDRESS, localAddr.toString()); } // Needs to be done after execute to pick up all the headers res.setRequestHeaders(getConnectionHeaders(httpMethod)); // Request sent. Now get the response: InputStream instream = httpMethod.getResponseBodyAsStream(); if (instream != null) {// will be null for HEAD instream = new CountingInputStream(instream); try { Header responseHeader = httpMethod.getResponseHeader(HTTPConstants.HEADER_CONTENT_ENCODING); if (responseHeader != null && HTTPConstants.ENCODING_GZIP.equals(responseHeader.getValue())) { InputStream tmpInput = new GZIPInputStream(instream); // tmp inputstream needs to have a good counting res.setResponseData( readResponse(res, tmpInput, (int) httpMethod.getResponseContentLength())); } else { res.setResponseData( readResponse(res, instream, (int) httpMethod.getResponseContentLength())); } } finally { JOrphanUtils.closeQuietly(instream); } } res.sampleEnd(); // Done with the sampling proper. // Now collect the results into the HTTPSampleResult: res.setSampleLabel(httpMethod.getURI().toString()); // Pick up Actual path (after redirects) res.setResponseCode(Integer.toString(statusCode)); res.setSuccessful(isSuccessCode(statusCode)); res.setResponseMessage(httpMethod.getStatusText()); String ct = null; Header h = httpMethod.getResponseHeader(HTTPConstants.HEADER_CONTENT_TYPE); if (h != null)// Can be missing, e.g. on redirect { ct = h.getValue(); res.setContentType(ct);// e.g. text/html; charset=ISO-8859-1 res.setEncodingAndType(ct); } res.setResponseHeaders(getResponseHeaders(httpMethod)); if (res.isRedirect()) { final Header headerLocation = httpMethod.getResponseHeader(HTTPConstants.HEADER_LOCATION); if (headerLocation == null) { // HTTP protocol violation, but avoids NPE throw new IllegalArgumentException("Missing location header"); } String redirectLocation = headerLocation.getValue(); res.setRedirectLocation(redirectLocation); // in case sanitising fails } // record some sizes to allow HTTPSampleResult.getBytes() with different options if (instream != null) { res.setBodySize(((CountingInputStream) instream).getCount()); } res.setHeadersSize(calculateHeadersSize(httpMethod)); if (log.isDebugEnabled()) { log.debug("Response headersSize=" + res.getHeadersSize() + " bodySize=" + res.getBodySize() + " Total=" + (res.getHeadersSize() + res.getBodySize())); } // If we redirected automatically, the URL may have changed if (getAutoRedirects()) { res.setURL(new URL(httpMethod.getURI().toString())); } // Store any cookies received in the cookie manager: saveConnectionCookies(httpMethod, res.getURL(), getCookieManager()); // Save cache information if (cacheManager != null) { cacheManager.saveDetails(httpMethod, res); } // Follow redirects and download page resources if appropriate: res = resultProcessing(areFollowingRedirect, frameDepth, res); log.debug("End : sample"); return res; } catch (IllegalArgumentException e) { // e.g. some kinds of invalid URL res.sampleEnd(); // pick up headers if failed to execute the request // httpMethod can be null if method is unexpected if (httpMethod != null) { res.setRequestHeaders(getConnectionHeaders(httpMethod)); } errorResult(e, res); return res; } catch (IOException e) { res.sampleEnd(); // pick up headers if failed to execute the request // httpMethod cannot be null here, otherwise // it would have been caught in the previous catch block res.setRequestHeaders(getConnectionHeaders(httpMethod)); errorResult(e, res); return res; } finally { savedClient = null; if (httpMethod != null) { httpMethod.releaseConnection(); } } }
From source file:org.apache.jmeter.protocol.oauth.sampler.OAuthSampler.java
/** * Samples the URL passed in and stores the result in * <code>HTTPSampleResult</code>, following redirects and downloading * page resources as appropriate.//from ww w .ja v a 2 s . co m * <p> * When getting a redirect target, redirects are not followed and resources * are not downloaded. The caller will take care of this. * * @param url * URL to sample * @param method * HTTP method: GET, POST,... * @param areFollowingRedirect * whether we're getting a redirect target * @param frameDepth * Depth of this target in the frame structure. Used only to * prevent infinite recursion. * @return results of the sampling */ protected HTTPSampleResult sample(URL url, String method, boolean areFollowingRedirect, int frameDepth) { String urlStr = url.toExternalForm(); // Check if this is an entity-enclosing method boolean isPost = method.equals(POST) || method.equals(PUT); HTTPSampleResult res = new HTTPSampleResult(); res.setMonitor(isMonitor()); // Handles OAuth signing try { message = getOAuthMessage(url, method); urlStr = message.URL; if (isPost) { urlStr = message.URL; } else { if (useAuthHeader) urlStr = OAuth.addParameters(message.URL, nonOAuthParams); else urlStr = OAuth.addParameters(message.URL, message.getParameters()); } } catch (IOException e) { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); //$NON-NLS-1$ return err; } catch (OAuthException e) { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); //$NON-NLS-1$ return err; } catch (URISyntaxException e) { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); //$NON-NLS-1$ return err; } log.debug("Start : sample " + urlStr); //$NON-NLS-1$ log.debug("method " + method); //$NON-NLS-1$ HttpMethodBase httpMethod = null; res.setSampleLabel(urlStr); // May be replaced later res.setHTTPMethod(method); res.sampleStart(); // Count the retries as well in the time HttpClient client = null; InputStream instream = null; try { // May generate IllegalArgumentException if (method.equals(POST)) { httpMethod = new PostMethod(urlStr); } else if (method.equals(PUT)) { httpMethod = new PutMethod(urlStr); } else if (method.equals(HEAD)) { httpMethod = new HeadMethod(urlStr); } else if (method.equals(TRACE)) { httpMethod = new TraceMethod(urlStr); } else if (method.equals(OPTIONS)) { httpMethod = new OptionsMethod(urlStr); } else if (method.equals(DELETE)) { httpMethod = new DeleteMethod(urlStr); } else if (method.equals(GET)) { httpMethod = new GetMethod(urlStr); } else { log.error("Unexpected method (converted to GET): " + method); //$NON-NLS-1$ httpMethod = new GetMethod(urlStr); } // Set any default request headers setDefaultRequestHeaders(httpMethod); // Setup connection client = setupConnection(new URL(urlStr), httpMethod, res); // Handle POST and PUT if (isPost) { String postBody = sendPostData(httpMethod); res.setQueryString(postBody); } res.setRequestHeaders(getConnectionHeaders(httpMethod)); int statusCode = client.executeMethod(httpMethod); // Request sent. Now get the response: instream = httpMethod.getResponseBodyAsStream(); if (instream != null) {// will be null for HEAD Header responseHeader = httpMethod.getResponseHeader(HEADER_CONTENT_ENCODING); if (responseHeader != null && ENCODING_GZIP.equals(responseHeader.getValue())) { instream = new GZIPInputStream(instream); } res.setResponseData(readResponse(res, instream, (int) httpMethod.getResponseContentLength())); } res.sampleEnd(); // Done with the sampling proper. // Now collect the results into the HTTPSampleResult: res.setSampleLabel(httpMethod.getURI().toString()); // Pick up Actual path (after redirects) res.setResponseCode(Integer.toString(statusCode)); res.setSuccessful(isSuccessCode(statusCode)); res.setResponseMessage(httpMethod.getStatusText()); String ct = null; org.apache.commons.httpclient.Header h = httpMethod.getResponseHeader(HEADER_CONTENT_TYPE); if (h != null)// Can be missing, e.g. on redirect { ct = h.getValue(); res.setContentType(ct);// e.g. text/html; charset=ISO-8859-1 res.setEncodingAndType(ct); } res.setResponseHeaders(getResponseHeaders(httpMethod)); if (res.isRedirect()) { final Header headerLocation = httpMethod.getResponseHeader(HEADER_LOCATION); if (headerLocation == null) { // HTTP protocol violation, but avoids NPE throw new IllegalArgumentException("Missing location header"); //$NON-NLS-1$ } res.setRedirectLocation(headerLocation.getValue()); } // If we redirected automatically, the URL may have changed if (getAutoRedirects()) { res.setURL(new URL(httpMethod.getURI().toString())); } // Store any cookies received in the cookie manager: saveConnectionCookies(httpMethod, res.getURL(), getCookieManager()); // Save cache information final CacheManager cacheManager = getCacheManager(); if (cacheManager != null) { cacheManager.saveDetails(httpMethod, res); } // Follow redirects and download page resources if appropriate: res = resultProcessing(areFollowingRedirect, frameDepth, res); log.debug("End : sample"); //$NON-NLS-1$ httpMethod.releaseConnection(); return res; } catch (IllegalArgumentException e)// e.g. some kinds of invalid URL { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); //$NON-NLS-1$ return err; } catch (IOException e) { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); //$NON-NLS-1$ return err; } finally { JOrphanUtils.closeQuietly(instream); if (httpMethod != null) { httpMethod.releaseConnection(); } } }
From source file:org.appcelerator.transport.ProxyTransportServlet.java
@Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String method = request.getMethod(); String url = request.getParameter("url"); if (url == null) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return;/*w ww . jav a 2s . c o m*/ } url = URLDecoder.decode(url, "UTF-8"); if (url.indexOf("://") == -1) { url = new String(Base64.decode(url)); } HttpClient client = new HttpClient(); HttpMethodBase methodBase = null; if (method.equalsIgnoreCase("POST")) { methodBase = new PostMethod(url); ByteArrayOutputStream out = new ByteArrayOutputStream(); Util.copy(request.getInputStream(), out); ByteArrayRequestEntity req = new ByteArrayRequestEntity(out.toByteArray()); ((PostMethod) methodBase).setRequestEntity(req); } else if (method.equalsIgnoreCase("GET")) { methodBase = new GetMethod(url); methodBase.setFollowRedirects(true); } else if (method.equalsIgnoreCase("PUT")) { methodBase = new PutMethod(url); ByteArrayOutputStream out = new ByteArrayOutputStream(); Util.copy(request.getInputStream(), out); ByteArrayRequestEntity req = new ByteArrayRequestEntity(out.toByteArray()); ((PutMethod) methodBase).setRequestEntity(req); } else if (method.equalsIgnoreCase("DELETE")) { methodBase = new DeleteMethod(url); } else if (method.equalsIgnoreCase("OPTIONS")) { methodBase = new OptionsMethod(url); } methodBase.setRequestHeader("User-Agent", request.getHeader("user-agent") + " (Appcelerator Proxy)"); if (LOG.isDebugEnabled()) { LOG.debug("Proxying url: " + url + ", method: " + method); } int status = client.executeMethod(methodBase); response.setStatus(status); response.setContentLength((int) methodBase.getResponseContentLength()); for (Header header : methodBase.getResponseHeaders()) { String name = header.getName(); if (name.equalsIgnoreCase("Set-Cookie") == false && name.equals("Transfer-Encoding") == false) { response.setHeader(name, header.getValue()); } } InputStream in = methodBase.getResponseBodyAsStream(); Util.copy(in, response.getOutputStream()); }