List of usage examples for org.apache.http.client.methods HttpUriRequest abort
void abort() throws UnsupportedOperationException;
From source file:jp.mixi.android.sdk.MixiContainerImpl.java
/** * ????//from w w w . j av a2 s . co m * * @param method ?http * @param listener ??? * @param doRetry ???? * @return ???? * @throws FileNotFoundException * @throws MalformedURLException * @throws IOException * @throws JSONException * @throws RemoteException * @throws ApiException */ private String request(HttpUriRequest method, final CallbackListener listener, boolean doRetry) throws FileNotFoundException, MalformedURLException, IOException, JSONException, RemoteException, ApiException { HttpClient client = getHttpClient(); HttpEntity entity = null; try { if (getAccessExpiresIn() < System.currentTimeMillis() + mSocketTimeout + mConnectionTimeout) { refreshToken(); } method.setHeaders(getCommonHeaders()); HttpResponse res = client.execute(method); StatusLine status = res.getStatusLine(); entity = res.getEntity(); // gzip?????? if (isGZipEntity(entity)) { entity = decompressesGZipEntity(entity); } String responseBody = null; switch (status.getStatusCode()) { case HttpStatus.SC_OK: case HttpStatus.SC_CREATED: // OK?string????? Log.v(TAG, "HTTP OK"); return EntityUtils.toString(entity, HTTP.UTF_8); case HttpStatus.SC_UNAUTHORIZED: if (isExpiredToken(res, status) && doRetry) { refreshToken(); entity.consumeContent(); entity = null; return request(method, listener, false); } responseBody = EntityUtils.toString(entity, HTTP.UTF_8); if (responseBody != null && responseBody.length() > 0) { throw new ApiException(status, responseBody); } else { throw new ApiException(status, status.getReasonPhrase()); } default: responseBody = EntityUtils.toString(entity, HTTP.UTF_8); if (responseBody != null && responseBody.length() > 0) { throw new ApiException(status, responseBody); } else { throw new ApiException(status, status.getReasonPhrase()); } } } finally { // if (entity != null) { entity.consumeContent(); } if (method != null) { method.abort(); } } }
From source file:org.apache.jena.fuseki.http.DatasetGraphAccessorHTTP.java
private Graph exec(String targetStr, Graph graphToSend, HttpUriRequest httpRequest, boolean processBody) { HttpClient httpclient = new SystemDefaultHttpClient(httpParams); if (graphToSend != null) { // ??? httpRequest isa Post // Impedence mismatch - is there a better way? ByteArrayOutputStream out = new ByteArrayOutputStream(); Model model = ModelFactory.createModelForGraph(graphToSend); model.write(out, "RDF/XML"); byte[] bytes = out.toByteArray(); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); InputStreamEntity reqEntity = new InputStreamEntity(in, bytes.length); reqEntity.setContentType(WebContent.contentTypeRDFXML); reqEntity.setContentEncoding(WebContent.charsetUTF8); HttpEntity entity = reqEntity;/*from w w w. j av a 2 s . c o m*/ ((HttpEntityEnclosingRequestBase) httpRequest).setEntity(entity); } TypedInputStream ts = null; // httpclient.getParams().setXXX try { HttpResponse response = httpclient.execute(httpRequest); int responseCode = response.getStatusLine().getStatusCode(); String responseMessage = response.getStatusLine().getReasonPhrase(); if (HttpSC.isRedirection(responseCode)) // Not implemented yet. throw FusekiRequestException.create(responseCode, responseMessage); // Other 400 and 500 - errors if (HttpSC.isClientError(responseCode) || HttpSC.isServerError(responseCode)) throw FusekiRequestException.create(responseCode, responseMessage); if (responseCode == HttpSC.NO_CONTENT_204) return null; if (responseCode == HttpSC.CREATED_201) return null; if (responseCode != HttpSC.OK_200) { Log.warn(this, "Unexpected status code"); throw FusekiRequestException.create(responseCode, responseMessage); } // May not have a body. String ct = getHeader(response, HttpNames.hContentType); if (ct == null) { HttpEntity entity = response.getEntity(); if (entity != null) { InputStream instream = entity.getContent(); // Read to completion? instream.close(); } return null; } // Tidy. See ConNeg / MediaType. String x = getHeader(response, HttpNames.hContentType); String y[] = x.split(";"); String contentType = null; if (y[0] != null) contentType = y[0].trim(); String charset = null; if (y.length > 1 && y[1] != null) charset = y[1].trim(); // Get hold of the response entity HttpEntity entity = response.getEntity(); if (entity != null) { InputStream instream = entity.getContent(); // String mimeType = ConNeg.chooseContentType(request, rdfOffer, ConNeg.acceptRDFXML).getAcceptType() ; // String charset = ConNeg.chooseCharset(request, charsetOffer, ConNeg.charsetUTF8).getAcceptType() ; ts = new TypedInputStream(instream, contentType, charset, null); } Graph graph = GraphFactory.createGraphMem(); if (processBody) readGraph(graph, ts, null); if (ts != null) ts.close(); Graph graph2 = new UnmodifiableGraph(graph); return graph2; } catch (IOException ex) { httpRequest.abort(); return null; } }
From source file:net.yacy.cora.protocol.http.HTTPClient.java
private void execute(final HttpUriRequest httpUriRequest, final boolean concurrent) throws IOException { final HttpClientContext context = HttpClientContext.create(); context.setRequestConfig(reqConfBuilder.build()); if (this.host != null) context.setTargetHost(new HttpHost(this.host)); setHeaders(httpUriRequest);/*ww w. j av a 2 s .c o m*/ // statistics storeConnectionInfo(httpUriRequest); // execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF if (httpUriRequest instanceof HttpEntityEnclosingRequest) { final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest; final HttpEntity entity = hrequest.getEntity(); assert entity != null; //assert !entity.isChunked(); //assert entity.getContentLength() >= 0; assert !hrequest.expectContinue(); } final String initialThreadName = Thread.currentThread().getName(); Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI()); final long time = System.currentTimeMillis(); try { if (concurrent) { FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>( new Callable<CloseableHttpResponse>() { @Override public CloseableHttpResponse call() throws ClientProtocolException, IOException { final CloseableHttpClient client = clientBuilder.build(); CloseableHttpResponse response = client.execute(httpUriRequest, context); return response; } }); executor.execute(t); try { this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw e.getCause(); } catch (Throwable e) { } try { t.cancel(true); } catch (Throwable e) { } if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + httpUriRequest.getURI().toString()); } else { final CloseableHttpClient client = clientBuilder.build(); this.httpResponse = client.execute(httpUriRequest, context); } this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); } catch (final Throwable e) { ConnectionInfo.removeConnection(httpUriRequest.hashCode()); httpUriRequest.abort(); if (this.httpResponse != null) this.httpResponse.close(); //e.printStackTrace(); throw new IOException( "Client can't execute: " + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage()) + " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + httpUriRequest.getURI().toString()); } finally { /* Restore the thread initial name */ Thread.currentThread().setName(initialThreadName); } }
From source file:illab.nabal.proxy.AbstractContext.java
/** * Execute a request and retrieve a response as a string value. * /* ww w .j a v a2 s . c o m*/ * @param request * @param isEmptyAllowed * @return responseString * @throws Exception */ protected synchronized String getResponseString(HttpUriRequest request, boolean isEmptyAllowed) throws Exception { String responseString = null; if (request != null) { AndroidHttpClient client = AndroidHttpClient.newInstance(SystemProperties.USER_AGENT_ANDROID, mContext); try { HttpResponse response = client.execute(request); HttpEntity entity = response.getEntity(); if (entity != null) { StringHelper strings = new StringHelper(); InputStream instream = entity.getContent(); int len; byte[] tmp = new byte[2048]; while ((len = instream.read(tmp)) != -1) { strings.append(new String(tmp, 0, len, UTF_8)); } responseString = strings.toString().trim(); Log.v(TAG, " ## HTTP response for " + request.getURI() + " : \n" + responseString); strings = null; entity.consumeContent(); } else { Log.w(TAG, "entity IS NULL"); } StatusLine statusLine = response.getStatusLine(); int statusCode = statusLine.getStatusCode(); if (statusCode > HttpStatus.SC_MULTIPLE_CHOICES) { Log.e(TAG, "HTTP status : " + statusCode + " - " + statusLine.getReasonPhrase()); JSONObject responseJson = null; try { responseJson = new JSONObject(responseString); } catch (Exception e) { // DO NOTHING } // if response came in JSON format if (responseJson != null) { Log.e(TAG, "responseJson.toString() : " + responseJson.toString()); NetworkException ne = new NetworkException(statusCode, statusLine.getReasonPhrase(), responseJson.toString()); ne.setResponseJson(responseJson); throw ne; } // if response didn't came in JSON format else { Log.e(TAG, "responseString : " + responseString); NetworkException ne = new NetworkException(statusCode, statusLine.getReasonPhrase(), responseString); ne.setResponseString(responseString); throw ne; } } } catch (Exception e) { request.abort(); Log.e(TAG, e.getMessage()); throw e; } finally { if (client != null) { client.close(); } } } // throw an exception if empty reponse is not allowed if (isEmptyAllowed == false && StringHelper.isEmpty(responseString) == true) { Log.e(TAG, "HTTP response is empty"); throw new NetworkException("HTTP response is empty."); } return responseString; }
From source file:com.android.aft.AFNetworkConnection.AFNetworkConnection.java
public final AFNetworkConnectionResult wget(AFNetworkConnectionRequest request) throws IllegalStateException, IOException, URISyntaxException, AFRestClientException { // Get the request URL if (request.url == null) { if (AFConfig.ERROR_LOGS_ENABLED) { Log.e(LOG_TAG, "retrieveStringFromService - Compulsory Parameter : request URL has not been set"); }/*www. ja v a2s . c o m*/ throw new AFCompulsoryParameterException("Request URL has not been set"); } if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Request url : " + request.url); } // Get the request method if (request.method != HttpMethod.Get && request.method != HttpMethod.Post && request.method != HttpMethod.Put && request.method != HttpMethod.Delete) { if (AFConfig.ERROR_LOGS_ENABLED) { Log.e(LOG_TAG, "retrieveStringFromService - Request method other than METHOD_GET, METHOD_POST are not implemented"); } throw new IllegalArgumentException( "retrieveStringFromService - Request method other than METHOD_GET, METHOD_POST are not implemented"); } if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Request method : " + request.method); } // Get the request parameters if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Request parameters (number) : " + ((request.parameters != null) ? request.parameters.size() : "")); } // Get the request headers if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Request headers (number) : " + ((request.headers != null) ? request.headers.size() : "")); } // Set http client HttpClient client = null; if (mHttpClient != null) { client = mHttpClient; configureHttpClient(client); } else if (request.client != null) client = request.client; else { if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Request user agent : " + request.userAgent); } client = AndroidHttpClient.newInstance(request.userAgent, request.context); } // Result object AFNetworkConnectionResult result = null; HttpUriRequest uri_request = buildUriRequest(request); try { HttpResponse response = makeRequest(client, uri_request, request); StatusLine status = response.getStatusLine(); if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "retrieveStringFromService - Response status : " + status.getStatusCode()); } while (status.getStatusCode() < HttpStatus.SC_OK || status.getStatusCode() >= HttpStatus.SC_MULTIPLE_CHOICES) { if (AFConfig.ERROR_LOGS_ENABLED) { Log.e(LOG_TAG, "retrieveStringFromService - Invalid response from server : " + status.toString()); } if (status.getStatusCode() == HttpStatus.SC_MOVED_TEMPORARILY || status.getStatusCode() == HttpStatus.SC_MOVED_PERMANENTLY) { final Header newLocation = response.getFirstHeader("Location"); if (AFConfig.INFO_LOGS_ENABLED) { Log.i(LOG_TAG, "New location : " + newLocation.getValue()); } if (request.checkResponse) { final String newLocationValue = newLocation.getValue(); if (request.mFollowRedirect) { request.url = newLocationValue; uri_request = buildUriRequest(request); response = makeRequest(client, uri_request, request); status = response.getStatusLine(); } else { throw new AFRestClientException(status.getStatusCode(), "New location : " + newLocation, newLocationValue); } } } if (request.checkResponse && status.getStatusCode() != HttpStatus.SC_OK) { String result_str = convertStream(response, request); throw new AFRestClientException(status.getStatusCode(), "Invalid response from server : " + status.toString(), null, result_str); } } // Read the response data if (request.readHttpResponse) { final String result_str = convertStream(response, request); result = new AFNetworkConnectionResult(request, response); result.mResult = result_str; } else { result = new AFNetworkConnectionResult(request, response); result.mHttpClient = client; result.mNeedToCloseClient = mHttpClient == null && request.client == null; } result.mHttpRequest = uri_request; if (AFConfig.INFO_LOGS_ENABLED) { if (!request.readHttpResponse) Log.i(LOG_TAG, "retrieveStringFromService - Result need to be read with result object data"); else if (!request.hasToStoreResultInFile()) Log.i(LOG_TAG, "retrieveStringFromService - Result from webservice : " + result.mResult); else Log.i(LOG_TAG, "retrieveStringFromService - Result store in file : " + request.storeResultFile); } } finally { if (uri_request != null && request.readHttpResponse) { uri_request.abort(); } if (mHttpClient == null && request.client == null && request.readHttpResponse) { if (AFConfig.DEBUG_LOGS_ENABLED) { Log.d(LOG_TAG, "Close android http client"); } ((AndroidHttpClient) client).close(); } } return result; }
From source file:com.bigdata.rdf.sail.webapp.TestNanoSparqlClient.java
/** * Connect to a SPARQL end point (GET or POST query only). * /*www . j av a 2s .c om*/ * @param opts * The connection options. * * @return The connection. * * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/619"> * RemoteRepository class should use application/x-www-form-urlencoded * for large POST requests </a> */ private HttpResponse doConnect(final ConnectOptions opts) throws Exception { /* * Generate the fully formed and encoded URL. */ final StringBuilder urlString = new StringBuilder(opts.serviceURL); ConnectOptions.addQueryParams(urlString, opts.requestParams); final boolean isLongRequestURL = urlString.length() > 1024; if (isLongRequestURL && opts.method.equals("POST") && opts.entity == null) { /* * URL is too long. Reset the URL to just the service endpoint and * use application/x-www-form-urlencoded entity instead. Only in * cases where there is not already a request entity (SPARQL query * and SPARQL update). */ urlString.setLength(0); urlString.append(opts.serviceURL); opts.entity = ConnectOptions.getFormEntity(opts.requestParams); } else if (isLongRequestURL && opts.method.equals("GET") && opts.entity == null) { /* * Convert automatically to a POST if the request URL is too long. * * Note: [opts.entity == null] should always be true for a GET so * this bit is a paranoia check. */ opts.method = "POST"; urlString.setLength(0); urlString.append(opts.serviceURL); opts.entity = ConnectOptions.getFormEntity(opts.requestParams); } if (log.isDebugEnabled()) { log.debug("*** Request ***"); log.debug(opts.serviceURL); log.debug(opts.method); log.debug("query=" + opts.getRequestParam("query")); log.debug(urlString.toString()); } HttpUriRequest request = null; try { request = RemoteRepository.newRequest(urlString.toString(), opts.method); if (opts.requestHeaders != null) { for (Map.Entry<String, String> e : opts.requestHeaders.entrySet()) { request.addHeader(e.getKey(), e.getValue()); if (log.isDebugEnabled()) log.debug(e.getKey() + ": " + e.getValue()); } } // // conn = doConnect(urlString.toString(), opts.method); // final URL url = new URL(urlString.toString()); // conn = (HttpURLConnection) url.openConnection(); // conn.setRequestMethod(opts.method); // conn.setDoOutput(true); // conn.setDoInput(true); // conn.setUseCaches(false); // conn.setReadTimeout(opts.timeout); // conn.setRequestProperty("Accept", opts.acceptHeader); // if (log.isDebugEnabled()) // log.debug("Accept: " + opts.acceptHeader); if (opts.entity != null) { // if (opts.data == null) // throw new AssertionError(); // final String contentLength = Integer.toString(opts.data.length); // conn.setRequestProperty("Content-Type", opts.contentType); // conn.setRequestProperty("Content-Length", contentLength); // if (log.isDebugEnabled()) { // log.debug("Content-Type: " + opts.contentType); // log.debug("Content-Length: " + contentLength); // } // final ByteArrayEntity entity = new ByteArrayEntity(opts.data); // entity.setContentType(opts.contentType); ((HttpEntityEnclosingRequestBase) request).setEntity(opts.entity); // final OutputStream os = conn.getOutputStream(); // try { // os.write(opts.data); // os.flush(); // } finally { // os.close(); // } } final HttpResponse response = m_httpClient.execute(request); return response; // // connect. // conn.connect(); // // return conn; } catch (Throwable t) { /* * If something goes wrong, then close the http connection. * Otherwise, the connection will be closed by the caller. */ try { if (request != null) request.abort(); // // clean up the connection resources // if (conn != null) // conn.disconnect(); } catch (Throwable t2) { // ignored. } throw new RuntimeException(opts.serviceURL + " : " + t, t); } }
From source file:org.apache.shindig.gadgets.http.BasicHttpFetcher.java
public HttpResponse fetch(org.apache.shindig.gadgets.http.HttpRequest request) throws GadgetException { HttpUriRequest httpMethod = null;// ww w . j a v a 2s . c om Preconditions.checkNotNull(request); final String methodType = request.getMethod(); final org.apache.http.HttpResponse response; final long started = System.currentTimeMillis(); // Break the request Uri to its components: Uri uri = request.getUri(); if (StringUtils.isEmpty(uri.getAuthority())) { throw new GadgetException(GadgetException.Code.INVALID_USER_DATA, "Missing domain name for request: " + uri, HttpServletResponse.SC_BAD_REQUEST); } if (StringUtils.isEmpty(uri.getScheme())) { throw new GadgetException(GadgetException.Code.INVALID_USER_DATA, "Missing schema for request: " + uri, HttpServletResponse.SC_BAD_REQUEST); } String[] hostparts = StringUtils.splitPreserveAllTokens(uri.getAuthority(), ':'); int port = -1; // default port if (hostparts.length > 2) { throw new GadgetException(GadgetException.Code.INVALID_USER_DATA, "Bad host name in request: " + uri.getAuthority(), HttpServletResponse.SC_BAD_REQUEST); } if (hostparts.length == 2) { try { port = Integer.parseInt(hostparts[1]); } catch (NumberFormatException e) { throw new GadgetException(GadgetException.Code.INVALID_USER_DATA, "Bad port number in request: " + uri.getAuthority(), HttpServletResponse.SC_BAD_REQUEST); } } String requestUri = uri.getPath(); // Treat path as / if set as null. if (uri.getPath() == null) { requestUri = "/"; } if (uri.getQuery() != null) { requestUri += '?' + uri.getQuery(); } // Get the http host to connect to. HttpHost host = new HttpHost(hostparts[0], port, uri.getScheme()); try { if ("POST".equals(methodType) || "PUT".equals(methodType)) { HttpEntityEnclosingRequestBase enclosingMethod = ("POST".equals(methodType)) ? new HttpPost(requestUri) : new HttpPut(requestUri); if (request.getPostBodyLength() > 0) { enclosingMethod .setEntity(new InputStreamEntity(request.getPostBody(), request.getPostBodyLength())); } httpMethod = enclosingMethod; } else if ("GET".equals(methodType)) { httpMethod = new HttpGet(requestUri); } else if ("HEAD".equals(methodType)) { httpMethod = new HttpHead(requestUri); } else if ("DELETE".equals(methodType)) { httpMethod = new HttpDelete(requestUri); } for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) { httpMethod.addHeader(entry.getKey(), StringUtils.join(entry.getValue(), ',')); } // Disable following redirects. if (!request.getFollowRedirects()) { httpMethod.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, false); } // HttpClient doesn't handle all cases when breaking url (specifically '_' in domain) // So lets pass it the url parsed: response = FETCHER.execute(host, httpMethod); if (response == null) { throw new IOException("Unknown problem with request"); } long now = System.currentTimeMillis(); if (now - started > slowResponseWarning) { slowResponseWarning(request, started, now); } return makeResponse(response); } catch (Exception e) { long now = System.currentTimeMillis(); // Find timeout exceptions, respond accordingly if (TIMEOUT_EXCEPTIONS.contains(e.getClass())) { LOG.info("Timeout for " + request.getUri() + " Exception: " + e.getClass().getName() + " - " + e.getMessage() + " - " + (now - started) + "ms"); return HttpResponse.timeout(); } LOG.log(Level.INFO, "Got Exception fetching " + request.getUri() + " - " + (now - started) + "ms", e); // Separate shindig error from external error throw new GadgetException(GadgetException.Code.INTERNAL_SERVER_ERROR, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } finally { // cleanup any outstanding resources.. if (httpMethod != null) try { httpMethod.abort(); } catch (UnsupportedOperationException e) { // ignore } } }
From source file:org.seasar.robot.client.http.HcHttpClient.java
protected ResponseData processHttpMethod(final String url, final HttpUriRequest httpRequest) { try {/* w w w . j av a2 s. c o m*/ processRobotsTxt(url); } catch (final RobotCrawlAccessException e) { if (logger.isInfoEnabled()) { final StringBuilder buf = new StringBuilder(); buf.append(e.getMessage()); if (e.getCause() != null) { buf.append(e.getCause().getMessage()); } logger.info(buf.toString()); } else if (logger.isDebugEnabled()) { logger.debug("Crawling Access Exception at " + url, e); } } // request header for (final Header header : requestHeaderList) { httpRequest.addHeader(header); } ResponseData responseData = null; InputStream inputStream = null; HttpEntity httpEntity = null; try { // get a content final HttpResponse response = executeHttpClient(httpRequest); httpEntity = response.getEntity(); final int httpStatusCode = response.getStatusLine().getStatusCode(); // redirect if (isRedirectHttpStatus(httpStatusCode)) { final Header locationHeader = response.getFirstHeader("location"); if (locationHeader == null) { logger.warn("Invalid redirect location at " + url); } else { responseData = new ResponseData(); responseData.setRedirectLocation(locationHeader.getValue()); return responseData; } } long contentLength = 0; String contentEncoding = Constants.UTF_8; if (httpEntity == null) { inputStream = new ByteArrayInputStream(new byte[0]); } else { final InputStream responseBodyStream = httpEntity.getContent(); final File outputFile = File.createTempFile("s2robot-HcHttpClient-", ".out"); DeferredFileOutputStream dfos = null; try { try { dfos = new DeferredFileOutputStream(responseBodyInMemoryThresholdSize, outputFile); StreamUtil.drain(responseBodyStream, dfos); dfos.flush(); } finally { IOUtils.closeQuietly(dfos); } } catch (final Exception e) { if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } throw e; } if (dfos.isInMemory()) { inputStream = new ByteArrayInputStream(dfos.getData()); contentLength = dfos.getData().length; if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } } else { inputStream = new TemporaryFileInputStream(outputFile); contentLength = outputFile.length(); } final Header contentEncodingHeader = httpEntity.getContentEncoding(); if (contentEncodingHeader != null) { contentEncoding = contentEncodingHeader.getValue(); } } String contentType = null; final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { contentType = contentTypeHeader.getValue(); final int idx = contentType.indexOf(';'); if (idx > 0) { contentType = contentType.substring(0, idx); } } // check file size if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength(contentType); if (contentLength > maxLength) { throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength + " byte. The url is " + url); } } responseData = new ResponseData(); responseData.setUrl(url); responseData.setCharSet(contentEncoding); if (httpRequest instanceof HttpHead) { responseData.setMethod(Constants.HEAD_METHOD); } else { responseData.setMethod(Constants.GET_METHOD); } responseData.setResponseBody(inputStream); responseData.setHttpStatusCode(httpStatusCode); for (final Header header : response.getAllHeaders()) { responseData.addMetaData(header.getName(), header.getValue()); } if (contentType == null) { responseData.setMimeType(defaultMimeType); } else { responseData.setMimeType(contentType); } final Header contentLengthHeader = response.getFirstHeader("Content-Length"); if (contentLengthHeader == null) { responseData.setContentLength(contentLength); } else { final String value = contentLengthHeader.getValue(); try { responseData.setContentLength(Long.parseLong(value)); } catch (final Exception e) { responseData.setContentLength(contentLength); } } final Header lastModifiedHeader = response.getFirstHeader("Last-Modified"); if (lastModifiedHeader != null) { final String value = lastModifiedHeader.getValue(); if (StringUtil.isNotBlank(value)) { final Date d = parseLastModified(value); if (d != null) { responseData.setLastModified(d); } } } if (responseData.getLastModified() == null) { responseData.setLastModified(new Date()); // set current time } return responseData; } catch (final UnknownHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Unknown host(" + e.getMessage() + "): " + url, e); } catch (final NoRouteToHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("No route to host(" + e.getMessage() + "): " + url, e); } catch (final ConnectException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Connection time out(" + e.getMessage() + "): " + url, e); } catch (final SocketException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Socket exception(" + e.getMessage() + "): " + url, e); } catch (final IOException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("I/O exception(" + e.getMessage() + "): " + url, e); } catch (final RobotSystemException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw e; } catch (final Exception e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotSystemException("Failed to access " + url, e); } finally { EntityUtils.consumeQuietly(httpEntity); } }
From source file:org.eweb4j.spiderman.plugin.util.PageFetcherImpl.java
/** * /*w ww .j a v a 2s.co m*/ * @date 2013-1-7 ?11:08:54 * @param toFetchURL * @return */ public FetchResult request(FetchRequest req) throws Exception { FetchResult fetchResult = new FetchResult(); HttpUriRequest request = null; HttpEntity entity = null; String toFetchURL = req.getUrl(); boolean isPost = false; try { if (Http.Method.GET.equalsIgnoreCase(req.getHttpMethod())) request = new HttpGet(toFetchURL); else if (Http.Method.POST.equalsIgnoreCase(req.getHttpMethod())) { request = new HttpPost(toFetchURL); isPost = true; } else if (Http.Method.PUT.equalsIgnoreCase(req.getHttpMethod())) request = new HttpPut(toFetchURL); else if (Http.Method.HEAD.equalsIgnoreCase(req.getHttpMethod())) request = new HttpHead(toFetchURL); else if (Http.Method.OPTIONS.equalsIgnoreCase(req.getHttpMethod())) request = new HttpOptions(toFetchURL); else if (Http.Method.DELETE.equalsIgnoreCase(req.getHttpMethod())) request = new HttpDelete(toFetchURL); else throw new Exception("Unknown http method name"); //???,?? // TODO ?delay? synchronized (mutex) { //?? long now = (new Date()).getTime(); //?Host?? if (now - lastFetchTime < config.getPolitenessDelay()) Thread.sleep(config.getPolitenessDelay() - (now - lastFetchTime)); //????HOST??URL lastFetchTime = (new Date()).getTime(); } //GZIP???GZIP? request.addHeader("Accept-Encoding", "gzip"); for (Iterator<Entry<String, String>> it = headers.entrySet().iterator(); it.hasNext();) { Entry<String, String> entry = it.next(); request.addHeader(entry.getKey(), entry.getValue()); } //? Header[] headers = request.getAllHeaders(); for (Header h : headers) { Map<String, List<String>> hs = req.getHeaders(); String key = h.getName(); List<String> val = hs.get(key); if (val == null) val = new ArrayList<String>(); val.add(h.getValue()); hs.put(key, val); } req.getCookies().putAll(this.cookies); fetchResult.setReq(req); HttpEntity reqEntity = null; if (Http.Method.POST.equalsIgnoreCase(req.getHttpMethod()) || Http.Method.PUT.equalsIgnoreCase(req.getHttpMethod())) { if (!req.getFiles().isEmpty()) { reqEntity = new MultipartEntity(HttpMultipartMode.BROWSER_COMPATIBLE); for (Iterator<Entry<String, List<File>>> it = req.getFiles().entrySet().iterator(); it .hasNext();) { Entry<String, List<File>> e = it.next(); String paramName = e.getKey(); for (File file : e.getValue()) { // For File parameters ((MultipartEntity) reqEntity).addPart(paramName, new FileBody(file)); } } for (Iterator<Entry<String, List<Object>>> it = req.getParams().entrySet().iterator(); it .hasNext();) { Entry<String, List<Object>> e = it.next(); String paramName = e.getKey(); for (Object paramValue : e.getValue()) { // For usual String parameters ((MultipartEntity) reqEntity).addPart(paramName, new StringBody( String.valueOf(paramValue), "text/plain", Charset.forName("UTF-8"))); } } } else { List<NameValuePair> params = new ArrayList<NameValuePair>(req.getParams().size()); for (Iterator<Entry<String, List<Object>>> it = req.getParams().entrySet().iterator(); it .hasNext();) { Entry<String, List<Object>> e = it.next(); String paramName = e.getKey(); for (Object paramValue : e.getValue()) { params.add(new BasicNameValuePair(paramName, String.valueOf(paramValue))); } } reqEntity = new UrlEncodedFormEntity(params, HTTP.UTF_8); } if (isPost) ((HttpPost) request).setEntity(reqEntity); else ((HttpPut) request).setEntity(reqEntity); } //?? HttpResponse response = httpClient.execute(request); headers = response.getAllHeaders(); for (Header h : headers) { Map<String, List<String>> hs = fetchResult.getHeaders(); String key = h.getName(); List<String> val = hs.get(key); if (val == null) val = new ArrayList<String>(); val.add(h.getValue()); hs.put(key, val); } //URL fetchResult.setFetchedUrl(toFetchURL); String uri = request.getURI().toString(); if (!uri.equals(toFetchURL)) if (!URLCanonicalizer.getCanonicalURL(uri).equals(toFetchURL)) fetchResult.setFetchedUrl(uri); entity = response.getEntity(); //??? int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { if (statusCode != HttpStatus.SC_NOT_FOUND) { Header locationHeader = response.getFirstHeader("Location"); //301?302?URL?? if (locationHeader != null && (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY)) fetchResult.setMovedToUrl( URLCanonicalizer.getCanonicalURL(locationHeader.getValue(), toFetchURL)); } //???OKURLstatusCode?? //??? if (this.site.getSkipStatusCode() != null && this.site.getSkipStatusCode().trim().length() > 0) { String[] scs = this.site.getSkipStatusCode().split(","); for (String code : scs) { int c = CommonUtil.toInt(code); //????entity if (statusCode == c) { assemPage(fetchResult, entity); break; } } } fetchResult.setStatusCode(statusCode); return fetchResult; } //?? if (entity != null) { fetchResult.setStatusCode(statusCode); assemPage(fetchResult, entity); return fetchResult; } } catch (Throwable e) { fetchResult.setFetchedUrl(e.toString()); fetchResult.setStatusCode(Status.INTERNAL_SERVER_ERROR.ordinal()); return fetchResult; } finally { try { if (entity == null && request != null) request.abort(); } catch (Exception e) { throw e; } } fetchResult.setStatusCode(Status.UNSPECIFIED_ERROR.ordinal()); return fetchResult; }