List of usage examples for org.apache.commons.httpclient HttpMethodBase getResponseContentLength
public long getResponseContentLength()
From source file:org.archive.crawler.fetcher.OptimizeFetchHTTP.java
protected void innerProcess(final CrawlURI curi) throws InterruptedException { if (!canFetch(curi)) { // Cannot fetch this, due to protocol, retries, or other problems return;/* ww w . j a v a 2 s . c o m*/ } HttpClient http = this.getClient(); setLocalIP(http); this.curisHandled++; // Note begin time curi.putLong(A_FETCH_BEGAN_TIME, System.currentTimeMillis()); // Get a reference to the HttpRecorder that is set into this ToeThread. HttpRecorder rec = HttpRecorder.getHttpRecorder(); // Shall we get a digest on the content downloaded? boolean digestContent = ((Boolean) getUncheckedAttribute(curi, ATTR_DIGEST_CONTENT)).booleanValue(); String algorithm = null; if (digestContent) { algorithm = ((String) getUncheckedAttribute(curi, ATTR_DIGEST_ALGORITHM)); rec.getRecordedInput().setDigest(algorithm); } else { // clear rec.getRecordedInput().setDigest((MessageDigest) null); } // Below we do two inner classes that add check of midfetch // filters just as we're about to receive the response body. String curiString = curi.getUURI().toString(); HttpMethodBase method = null; if (curi.isPost()) { method = new HttpRecorderPostMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } else { method = new HttpRecorderGetMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } HostConfiguration customConfigOrNull = configureMethod(curi, method); // Set httpRecorder into curi. Subsequent code both here and later // in extractors expects to find the HttpRecorder in the CrawlURI. curi.setHttpRecorder(rec); // Populate credentials. Set config so auth. is not automatic. boolean addedCredentials = populateCredentials(curi, method); method.setDoAuthentication(addedCredentials); // set hardMax on bytes (if set by operator) long hardMax = getMaxLength(curi); // set overall timeout (if set by operator) long timeoutMs = 1000 * getTimeout(curi); // Get max fetch rate (bytes/ms). It comes in in KB/sec long maxRateKBps = getMaxFetchRate(curi); rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps); try { http.executeMethod(customConfigOrNull, method); } catch (RecorderTooMuchHeaderException ex) { // when too much header material, abort like other truncations doAbort(curi, method, HEADER_TRUNC); } catch (IOException e) { failedExecuteCleanup(method, curi, e); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions in native // code... see // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException failedExecuteCleanup(method, curi, e); return; } // set softMax on bytes to get (if implied by content-length) long softMax = method.getResponseContentLength(); try { if (!curi.isSeed() && curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) { logger.debug(curi.getUURI().toString() + " is not modify"); curi.skipToProcessorChain(getController().getPostprocessorChain()); } else if (!method.isAborted()) { // Force read-to-end, so that any socket hangs occur here, // not in later modules. rec.getRecordedInput().readFullyOrUntil(softMax); } } catch (RecorderTimeoutException ex) { doAbort(curi, method, TIMER_TRUNC); } catch (RecorderLengthExceededException ex) { doAbort(curi, method, LENGTH_TRUNC); } catch (IOException e) { cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions from native code // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } finally { // ensure recording has stopped rec.closeRecorders(); logger.debug("cloase backup file.&uri= " + curi.getCrawlURIString()); if (!method.isAborted()) { method.releaseConnection(); } // Note completion time curi.putLong(A_FETCH_COMPLETED_TIME, System.currentTimeMillis()); // Set the response charset into the HttpRecord if available. setCharacterEncoding(rec, method); setSizes(curi, rec); } if (digestContent) { curi.setContentDigest(algorithm, rec.getRecordedInput().getDigestValue()); } logger.info((curi.isPost() ? "POST" : "GET") + " " + curi.getUURI().toString() + " " + method.getStatusCode() + " " + rec.getRecordedInput().getSize() + " " + curi.getContentType()); if (curi.isSuccess() && addedCredentials) { // Promote the credentials from the CrawlURI to the CrawlServer // so they are available for all subsequent CrawlURIs on this // server. promoteCredentials(curi); if (logger.isDebugEnabled()) { // Print out the cookie. Might help with the debugging. Header setCookie = method.getResponseHeader("set-cookie"); if (setCookie != null) { logger.debug(setCookie.toString().trim()); } } } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { // 401 is not 'success'. handle401(method, curi); } if (rec.getRecordedInput().isOpen()) { logger.error(curi.toString() + " RIS still open. Should have" + " been closed by method release: " + Thread.currentThread().getName()); try { rec.getRecordedInput().close(); } catch (IOException e) { logger.error("second-chance RIS close failed", e); } } }
From source file:org.jboss.mod_cluster.Client.java
public int runit() throws Exception { PostMethod pm = null;//w w w .j a v a 2s .c o m GetMethod gm = null; HttpMethodBase bm = null; long starttime, endtime; if (httpClient == null) httpClient = new HttpClient(); if (fd != null) { pm = new PostMethod(URL); // InputStreamRequestEntity buf = new InputStreamRequestEntity(fd); // XXX: Ugly hack to test... byte[] buffet = new byte[6144]; for (int i = 0; i < buffet.length; i++) buffet[i] = 'a'; ByteArrayRequestEntity buf = new ByteArrayRequestEntity(buffet); pm.setRequestEntity(buf); // pm.setRequestBody(fd); pm.setHttp11(true); pm.setContentChunked(true); // pm.setRequestContentLength(PostMethod.CONTENT_LENGTH_CHUNKED); bm = pm; } else if (post != null) { pm = new PostMethod(URL); pm.setRequestEntity(new StringRequestEntity(post, "application/x-www-form-urlencoded", "UTF8")); bm = pm; } else { gm = new GetMethod(URL); bm = gm; } if (user != null) { Credentials cred = new UsernamePasswordCredentials(user, pass); httpClient.getState().setCredentials(org.apache.commons.httpclient.auth.AuthScope.ANY, cred); } // System.out.println("Connecting to " + URL); Integer connectionTimeout = 40000; bm.getParams().setParameter("http.socket.timeout", connectionTimeout); bm.getParams().setParameter("http.connection.timeout", connectionTimeout); if (VirtualHost != null) bm.getParams().setVirtualHost(VirtualHost); httpClient.getParams().setParameter("http.socket.timeout", connectionTimeout); httpClient.getParams().setParameter("http.connection.timeout", connectionTimeout); if (jsessionid != null) { // System.out.println("jsessionid: " + jsessionid); bm.setRequestHeader("Cookie", "JSESSIONID=" + jsessionid); } try { if (gm == null) { pm.getParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY); starttime = System.currentTimeMillis(); httpResponseCode = httpClient.executeMethod(pm); endtime = System.currentTimeMillis(); } else { gm.getParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY); starttime = System.currentTimeMillis(); httpResponseCode = httpClient.executeMethod(gm); endtime = System.currentTimeMillis(); } if (httpResponseCode == 200) { response = bm.getResponseBodyAsString(); Cookie[] cookies = httpClient.getState().getCookies(); // System.out.println( "Cookies: " + cookies); if (cookies != null && cookies.length != 0) { for (int i = 0; i < cookies.length; i++) { Cookie cookie = cookies[i]; // System.out.println( "Cookie: " + cookie.getName() + ", Value: " + cookie.getValue()); if (cookie.getName().equals("JSESSIONID")) { if (jsessionid == null) { jsessionid = cookie.getValue(); String nodes[] = jsessionid.split("\\."); if (nodes.length == 2) node = nodes[1]; System.out.println("cookie first time: " + jsessionid); bm.releaseConnection(); return 0; // first time ok. } else { if (jsessionid.compareTo(cookie.getValue()) == 0) { if (logok) if (bm.getResponseHeader("Date") != null) System.out.println("cookie ok: " + bm.getResponseHeader("Date").toString().replace('\r', ' ') .replace('\n', ' ') + " response time: " + (endtime - starttime)); else { DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); System.out.println("cookie ok: " + dateFormat.format(date) + " response time: " + (endtime - starttime)); } bm.releaseConnection(); return 0; } else { System.out.println( "cookie \"second\" time: " + cookie.getValue() + " : " + jsessionid); System.out.println("cookie changed"); bm.releaseConnection(); if (checkcookie) return -1; else if (checknode) { String nodes[] = cookie.getValue().split("\\."); if (nodes.length != 2) { System.out.println("Can't find node in cookie"); return -1; } if (nodes[1].compareTo(node) == 0) { return 0; } else { System.out.println("node " + nodes[1] + " changed too"); return -1; } } else return 0; } } } } } else { // Look in the response to make sure that there is a cookie. int len = (int) bm.getResponseContentLength(); if (jsessionid != null && bm.getResponseBodyAsString(len).indexOf(jsessionid) != -1) { bm.releaseConnection(); return 0; } if (jsessionid == null && !checkcookie) { return 0; } System.out.println("No cookies"); } Header head = bm.getResponseHeader("getRequestedSessionId"); if (head != null) { HeaderElement[] heade = head.getElements(); requestedSessionId = heade[0].getValue(); } else { requestedSessionId = null; } } else { System.out.println("response: " + httpResponseCode); System.out.println("response: " + bm.getStatusLine()); response = bm.getResponseBodyAsString(); System.out.println("response: " + response); success = false; httpClient = null; } // System.out.println("response:\n" + bm.getResponseBodyAsString(len)); } catch (HttpException e) { e.printStackTrace(); success = false; httpClient = null; } System.out.println("DONE: " + httpResponseCode); bm.releaseConnection(); return httpResponseCode; }
From source file:org.springfield.mojo.http.HttpHelper.java
/** * Sends a standard HTTP request to the specified URI using the determined method. * Attaches the content, uses the specified content type, sets cookies, timeout and * request headers//from w ww . ja v a 2 s.c o m * * @param method - the request method * @param uri - the uri to request * @param body - the content * @param contentType - the content type * @param cookies - cookies * @param timeout - timeout in milliseconds * @param charSet - the character set * @param requestHeaders - extra user defined headers * @return response */ public static Response sendRequest(String method, String uri, String body, String contentType, String cookies, int timeout, String charSet, Map<String, String> requestHeaders) { // http client HttpClient client = new HttpClient(); // method HttpMethodBase reqMethod = null; if (method.equals(HttpMethods.PUT)) { reqMethod = new PutMethod(uri); } else if (method.equals(HttpMethods.POST)) { reqMethod = new PostMethod(uri); } else if (method.equals(HttpMethods.GET)) { if (body != null) { // hack to be able to send a request body with a get (only if required) reqMethod = new PostMethod(uri) { public String getName() { return "GET"; } }; } else { reqMethod = new GetMethod(uri); } } else if (method.equals(HttpMethods.DELETE)) { if (body != null) { // hack to be able to send a request body with a delete (only if required) reqMethod = new PostMethod(uri) { public String getName() { return "DELETE"; } }; } else { reqMethod = new DeleteMethod(uri); } } else if (method.equals(HttpMethods.HEAD)) { reqMethod = new HeadMethod(uri); } else if (method.equals(HttpMethods.TRACE)) { reqMethod = new TraceMethod(uri); } else if (method.equals(HttpMethods.OPTIONS)) { reqMethod = new OptionsMethod(uri); } // add request body if (body != null) { try { RequestEntity entity = new StringRequestEntity(body, contentType, charSet); ((EntityEnclosingMethod) reqMethod).setRequestEntity(entity); reqMethod.setRequestHeader("Content-type", contentType); } catch (Exception e) { e.printStackTrace(); } } // add cookies if (cookies != null) { reqMethod.addRequestHeader("Cookie", cookies); } // add custom headers if (requestHeaders != null) { for (Map.Entry<String, String> header : requestHeaders.entrySet()) { String name = header.getKey(); String value = header.getValue(); reqMethod.addRequestHeader(name, value); } } Response response = new Response(); // do request try { if (timeout != -1) { client.getParams().setSoTimeout(timeout); } int statusCode = client.executeMethod(reqMethod); response.setStatusCode(statusCode); } catch (Exception e) { e.printStackTrace(); } // read response try { InputStream instream = reqMethod.getResponseBodyAsStream(); ByteArrayOutputStream outstream = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int len; while ((len = instream.read(buffer)) > 0) { outstream.write(buffer, 0, len); } String resp = new String(outstream.toByteArray(), reqMethod.getResponseCharSet()); response.setResponse(resp); //set content length long contentLength = reqMethod.getResponseContentLength(); response.setContentLength(contentLength); //set character set String respCharSet = reqMethod.getResponseCharSet(); response.setCharSet(respCharSet); //set all headers Header[] headers = reqMethod.getResponseHeaders(); response.setHeaders(headers); } catch (Exception e) { e.printStackTrace(); } // release connection reqMethod.releaseConnection(); // return return response; }
From source file:smartrics.jmeter.sampler.RestSampler.java
/** * Method invoked by JMeter when a sample needs to happen. It's actually an * indirect call from the main sampler interface. it's resolved in the base * class./*from ww w. j ava 2 s . c o m*/ * * This is a copy and paste from the HTTPSampler2 - quick and dirty hack as * that class is not very extensible. The reason to extend and slightly * modify is that I needed to get the body content from a text field in the * GUI rather than a file. */ protected HTTPSampleResult sample(URL url, String method, boolean areFollowingRedirect, int frameDepth) { String urlStr = url.toString(); log.debug("Start : sample " + urlStr); log.debug("method " + method); HttpMethodBase httpMethod = null; HTTPSampleResult res = new HTTPSampleResult(); res.setMonitor(isMonitor()); res.setSampleLabel(urlStr); // May be replaced later res.setHTTPMethod(method); res.setURL(url); res.sampleStart(); // Count the retries as well in the time HttpClient client = null; InputStream instream = null; try { httpMethod = createHttpMethod(method, urlStr); // Set any default request headers res.setRequestHeaders(""); // Setup connection client = setupConnection(url, httpMethod, res); // Handle the various methods if (httpMethod instanceof EntityEnclosingMethod) { String postBody = sendData((EntityEnclosingMethod) httpMethod); res.setResponseData(postBody.getBytes()); } overrideHeaders(httpMethod); res.setRequestHeaders(getConnectionHeaders(httpMethod)); int statusCode = -1; try { statusCode = client.executeMethod(httpMethod); } catch (RuntimeException e) { log.error("Exception when executing '" + httpMethod + "'", e); throw e; } // Request sent. Now get the response: instream = httpMethod.getResponseBodyAsStream(); if (instream != null) {// will be null for HEAD Header responseHeader = httpMethod.getResponseHeader(HEADER_CONTENT_ENCODING); if (responseHeader != null && ENCODING_GZIP.equals(responseHeader.getValue())) { instream = new GZIPInputStream(instream); } res.setResponseData(readResponse(res, instream, (int) httpMethod.getResponseContentLength())); } res.sampleEnd(); // Done with the sampling proper. // Now collect the results into the HTTPSampleResult: res.setSampleLabel(httpMethod.getURI().toString()); // Pick up Actual path (after redirects) res.setResponseCode(Integer.toString(statusCode)); res.setSuccessful(isSuccessCode(statusCode)); res.setResponseMessage(httpMethod.getStatusText()); String ct = null; org.apache.commons.httpclient.Header h = httpMethod.getResponseHeader(HEADER_CONTENT_TYPE); if (h != null)// Can be missing, e.g. on redirect { ct = h.getValue(); res.setContentType(ct);// e.g. text/html; charset=ISO-8859-1 res.setEncodingAndType(ct); } String responseHeaders = getResponseHeaders(httpMethod); res.setResponseHeaders(responseHeaders); if (res.isRedirect()) { final Header headerLocation = httpMethod.getResponseHeader(HEADER_LOCATION); if (headerLocation == null) { // HTTP protocol violation, but // avoids NPE throw new IllegalArgumentException("Missing location header"); } res.setRedirectLocation(headerLocation.getValue()); } // If we redirected automatically, the URL may have changed if (getAutoRedirects()) { res.setURL(new URL(httpMethod.getURI().toString())); } // Store any cookies received in the cookie manager: saveConnectionCookies(httpMethod, res.getURL(), getCookieManager()); // Save cache information final CacheManager cacheManager = getCacheManager(); if (cacheManager != null) { cacheManager.saveDetails(httpMethod, res); } // Follow redirects and download page resources if appropriate: res = resultProcessing(areFollowingRedirect, frameDepth, res); log.debug("End : sample"); httpMethod.releaseConnection(); return res; } catch (IllegalArgumentException e)// e.g. some kinds of invalid URL { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); return err; } catch (IOException e) { res.sampleEnd(); HTTPSampleResult err = errorResult(e, res); err.setSampleLabel("Error: " + url.toString()); return err; } finally { JOrphanUtils.closeQuietly(instream); if (httpMethod != null) { httpMethod.releaseConnection(); } } }