List of usage examples for org.apache.commons.httpclient HttpMethod setFollowRedirects
public abstract void setFollowRedirects(boolean paramBoolean);
From source file:ait.ffma.service.preservation.riskmanagement.TestLODFreebaseRetrieve.java
/** * This method evaluates Freebase description for passed guid number. * @param guid/* w w w . j ava2 s .c om*/ * @return description text */ public String getDescription(String guid) { String res = ""; if (guid != null) { guid = guid.replaceAll(LODConstants.STRING_SEPARATOR, ""); HttpClient client = new HttpClient(); client.getHttpConnectionManager().getParams().setConnectionTimeout(10000); String url = "http://hotshot.dfhuynh.user.dev.freebaseapps.com/html?id=/guid/" + guid; HttpMethod method = new GetMethod(url); method.setFollowRedirects(false); try { client.executeMethod(method); String responseBody = method.getResponseBodyAsString(); log.info("Response body: " + responseBody); String descriptionStr = responseBody.substring( responseBody.indexOf(LODConstants.DESC_BEGIN) + LODConstants.DESC_BEGIN.length(), responseBody.indexOf(LODConstants.DESC_P_END)); res = descriptionStr.replaceAll(LODConstants.DESC_P, ""); } catch (HttpException he) { log.fine("Http error connecting to " + url + ""); } catch (IOException ioe) { log.fine("Unable to connect to " + url + ""); } catch (Exception e) { log.fine("Description retrieval exception. Unable to parse response from " + url + " (Exception: " + e.getLocalizedMessage() + ")"); } method.releaseConnection(); } return res; }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
/** * Configure the HttpMethod setting options and headers. * * @param curi CrawlURI from which we pull configuration. * @param method The Method to configure. * @return HostConfiguration copy customized for this CrawlURI */// www . jav a 2s .c om protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); // // set soTimeout // method.getParams().setSoTimeout( // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) // .intValue()); // Set cookie policy. method.getParams() .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue()) ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); // Use only HTTP/1.0 (to avoid receiving chunked responses) method.getParams().setVersion(HttpVersion.HTTP_1_0); CrawlOrder order = getSettingsHandler().getOrder(); String userAgent = curi.getUserAgent(); if (userAgent == null) { userAgent = order.getUserAgent(curi); } method.setRequestHeader("User-Agent", userAgent); method.setRequestHeader("From", order.getFrom(curi)); // Set retry handler. method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler()); final long maxLength = getMaxLength(curi); if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) { method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1))); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) { method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue()) { // RFC2616 says no referer header if referer is https and the url // is not String via = curi.flattenVia(); if (via != null && via.length() > 0 && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) { method.setRequestHeader(REFERER, via); } } if (!curi.isPrerequisite()) { setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since"); setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match"); } // TODO: What happens if below method adds a header already // added above: e.g. Connection, Range, or Referer? setAcceptHeaders(curi, method); HostConfiguration config = new HostConfiguration(http.getHostConfiguration()); configureProxy(curi, config); configureBindAddress(curi, config); return config; }
From source file:de.ingrid.portal.portlets.SearchResultPortlet.java
public void doView(javax.portlet.RenderRequest request, javax.portlet.RenderResponse response) throws PortletException, IOException { Context context = getContext(request); IngridResourceBundle messages = new IngridResourceBundle( getPortletConfig().getResourceBundle(request.getLocale()), request.getLocale()); context.put("MESSAGES", messages); context.put("Codelists", CodeListServiceFactory.instance()); context.put("enableFacete", PortalConfig.getInstance().getBoolean(PortalConfig.PORTAL_ENABLE_SEARCH_FACETE, false)); // add request language, used to localize the map client context.put("languageCode", request.getLocale().getLanguage()); PortletSession ps = request.getPortletSession(); if (request.getParameter("filter") != null) { if (request.getParameter("filter").equals("domain")) { context.put("isFilterDomain", true); }//from www .java 2 s. c o m } // ---------------------------------- // check for passed RENDER PARAMETERS (for bookmarking) and // ADAPT OUR PERMANENT STATE VIA MESSAGES // ---------------------------------- // starthit RANKED // WHEN NO GROUPING !!! String reqParam = null; int rankedStartHit = 0; try { reqParam = request.getParameter(Settings.PARAM_STARTHIT_RANKED); if (SearchState.adaptSearchState(request, Settings.PARAM_STARTHIT_RANKED, reqParam)) { rankedStartHit = (new Integer(reqParam)).intValue(); } } catch (Exception ex) { if (log.isErrorEnabled()) { log.error( "Problems parsing RANKED starthit from render request, set RANKED starthit to 0 ! reqParam=" + reqParam, ex); } } // WHEN GROUPING !!! // get the current page number, default to 1 int currentSelectorPage = 1; try { reqParam = request.getParameter(Settings.PARAM_CURRENT_SELECTOR_PAGE); if (SearchState.adaptSearchState(request, Settings.PARAM_CURRENT_SELECTOR_PAGE, reqParam)) { currentSelectorPage = new Integer(reqParam).intValue(); } } catch (Exception ex) { if (log.isErrorEnabled()) { log.error( "Problems parsing currentSelectorPage from render request, set currentSelectorPage to 1 ! reqParam=" + reqParam, ex); } } // starthit UNRANKED // NO GROUPING !!! // Do we need to read unrankedStartHit, is passed from navi when not grouped but UNRANKED SEARCH IS ALWAYS GROUPED !!! int unrankedStartHit = 0; try { reqParam = request.getParameter(Settings.PARAM_STARTHIT_UNRANKED); if (SearchState.adaptSearchState(request, Settings.PARAM_STARTHIT_UNRANKED, reqParam)) { unrankedStartHit = (new Integer(reqParam)).intValue(); } } catch (Exception ex) { if (log.isErrorEnabled()) { log.error( "Problems parsing UNRANKED starthit from render request, set UNRANKED starthit to 0 ! reqParam=" + reqParam, ex); } } // GROUPING ONLY !!! // UNRANKED SEARCH IS ALWAYS GROUPED !!! // get the current page number, default to 1 int currentSelectorPageUnranked = 1; try { reqParam = request.getParameter(Settings.PARAM_CURRENT_SELECTOR_PAGE_UNRANKED); if (SearchState.adaptSearchState(request, Settings.PARAM_CURRENT_SELECTOR_PAGE_UNRANKED, reqParam)) { currentSelectorPageUnranked = new Integer(reqParam).intValue(); } } catch (Exception ex) { if (log.isErrorEnabled()) { log.error( "Problems parsing currentSelectorPageUnranked from render request, set currentSelectorPageUnranked to 1 ! reqParam=" + reqParam, ex); } } // indicates whether we do a query or we read results from cache String queryType = (String) SearchState.getSearchStateObject(request, Settings.MSG_QUERY_EXECUTION_TYPE); if (queryType == null) { queryType = ""; } // get the filter from REQUEST, not search state ! -> so back button works ! // ALSO ADAPT SEARCH STATE, so following steps (query pre processor) work ! String filter = request.getParameter(Settings.PARAM_FILTER); if (filter != null && filter.length() == 0) { filter = null; } SearchState.adaptSearchState(request, Settings.PARAM_FILTER, filter); String filterSubject = request.getParameter(Settings.PARAM_SUBJECT); SearchState.adaptSearchState(request, Settings.PARAM_SUBJECT, filterSubject); // set filter params into context for filter display if (filter != null) { context.put("filteredBy", filter); if (filter.equals(Settings.PARAMV_GROUPING_PARTNER)) { context.put("filterSubject", UtilsSearch.mapResultValue(Settings.RESULT_KEY_PARTNER, filterSubject, null)); } else if (filter.equals(Settings.PARAMV_GROUPING_PROVIDER)) { context.put("filterSubject", UtilsSearch.mapResultValue(Settings.RESULT_KEY_PROVIDER, filterSubject, null)); } else if (filter.equals(Settings.PARAMV_GROUPING_PLUG_ID)) { context.put("filterSubject", UtilsSearch.mapResultValue(Settings.RESULT_KEY_PLUG_ID, filterSubject, null)); } else if (filter.equals(Settings.PARAMV_GROUPING_DOMAIN)) { String[] keyValuePair = UtilsSearch.getDomainKeyValuePair(filterSubject); String domainKey = keyValuePair[0]; String domainValue = keyValuePair[1]; // domain can be plugid or site or ... // we extract the according value from our subject if (domainKey.equals(Settings.QFIELD_PLUG_ID)) { domainValue = UtilsSearch.mapResultValue(Settings.RESULT_KEY_PLUG_ID, domainValue, null); } context.put("filterSubject", domainValue); } } // datasource from state (set in SimpleSearch Portlet) String selectedDS = SearchState.getSearchStateObjectAsString(request, Settings.PARAM_DATASOURCE); // IngridQuery from state (set in SimpleSearch Portlet) IngridQuery query = (IngridQuery) SearchState.getSearchStateObject(request, Settings.MSG_QUERY); // change datasource dependent from query input selectedDS = UtilsSearch.determineFinalPortalDatasource(selectedDS, query); // ---------------------------------- // set initial view template // ---------------------------------- // if no query set display "nothing" if (query == null) { setDefaultViewPage(TEMPLATE_NO_QUERY_SET); super.doView(request, response); return; } // selected data source ("Umweltinfo", Adressen" or "Forschungsprojekte") if (selectedDS == null) { selectedDS = Settings.SEARCH_INITIAL_DATASOURCE; } if (selectedDS.equals(Settings.PARAMV_DATASOURCE_ADDRESS)) { setDefaultViewPage(TEMPLATE_RESULT_ADDRESS); } else { setDefaultViewPage(TEMPLATE_RESULT); // default: right column IS grouped (by plugid) // set in context e.g. to show grouped navigation context.put("grouping_right", new Boolean(true)); if (filter != null) { // set one column result template if "Zeige alle ..." of plug or domain if (filter.equals(Settings.PARAMV_GROUPING_PLUG_ID) || filter.equals(Settings.PARAMV_GROUPING_DOMAIN)) { setDefaultViewPage(TEMPLATE_RESULT_FILTERED_ONECOLUMN); // only one column to render we switch off grouping_right to show ungrouped navigation ! context.put("grouping_right", new Boolean(false)); } } } // check whether right column is switched OFF and set according context flag boolean rightColumnDisabled = !PortalConfig.getInstance() .getBoolean(PortalConfig.PORTAL_ENABLE_SEARCH_RESULTS_UNRANKED, true); context.put("RIGHT_COLUMN_DISABLED", rightColumnDisabled); String currentView = getDefaultViewPage(); // ---------------------------------- // business logic // ---------------------------------- // check for Javascript boolean hasJavaScript = Utils.isJavaScriptEnabled(request); // determine whether we have to render only one result column boolean renderResultColumnRanked = true; boolean renderResultColumnUnranked = true; reqParam = request.getParameter("js_ranked"); // finally disable query for right column if switched OFF ! if (rightColumnDisabled) { renderResultColumnUnranked = false; } // create threaded query controller ThreadedQueryController controller = new ThreadedQueryController(); controller.setTimeout(PortalConfig.getInstance().getInt(PortalConfig.QUERY_TIMEOUT_THREADED, 120000)); QueryDescriptor qd = null; // store query in session UtilsSearch.addQueryToHistory(request); // RANKED IngridHitsWrapper rankedHits = null; if (renderResultColumnRanked) { if (reqParam == null) { // check if query must be executed if (queryType.equals(Settings.MSGV_NO_QUERY) || queryType.equals(Settings.MSGV_UNRANKED_QUERY)) { rankedHits = (IngridHitsWrapper) SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_RESULT_RANKED); if (log.isDebugEnabled()) { log.debug("Read RANKED hits from CACHE !!! rankedHits=" + rankedHits); } } else { // process query, create QueryDescriptor qd = QueryPreProcessor.createRankedQueryDescriptor(request); if (qd != null) { controller.addQuery("ranked", qd); SearchState.resetSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_RANKED); } } } } else { if (queryType.equals(Settings.MSGV_UNRANKED_QUERY)) { if (ps.getAttribute("isPagingUnranked") != null && ps.getAttribute("isPagingUnranked").equals(currentSelectorPageUnranked + "")) { ps.removeAttribute("isPagingUnranked"); rankedHits = (IngridHitsWrapper) SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_RESULT_RANKED); if (log.isDebugEnabled()) { log.debug("Read RANKED hits from CACHE !!! rankedHits=" + rankedHits); } } else { // process query, create QueryDescriptor qd = QueryPreProcessor.createRankedQueryDescriptor(request); if (qd != null) { controller.addQuery("ranked", qd); SearchState.resetSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_RANKED); } } } } // UNRANKED IngridHitsWrapper unrankedHits = null; if (renderResultColumnUnranked) { if (reqParam != null && reqParam.equals("false") || !hasJavaScript) { if (!currentView.equals(TEMPLATE_RESULT_ADDRESS)) { // check if query must be executed if (queryType.equals(Settings.MSGV_NO_QUERY) || queryType.equals(Settings.MSGV_RANKED_QUERY)) { if (!currentView.equals(TEMPLATE_RESULT_FILTERED_ONECOLUMN)) { unrankedHits = (IngridHitsWrapper) SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_RESULT_UNRANKED); if (log.isDebugEnabled()) { log.debug("Read UNRANKED hits from CACHE !!!! unrankedHits=" + unrankedHits); } } else { // process query, create QueryDescriptor qd = QueryPreProcessor.createUnrankedQueryDescriptor(request); if (qd != null) { controller.addQuery("unranked", qd); SearchState.resetSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_UNRANKED); } } } else { // process query, create QueryDescriptor qd = QueryPreProcessor.createUnrankedQueryDescriptor(request); if (qd != null) { controller.addQuery("unranked", qd); SearchState.resetSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_UNRANKED); } } } } else { if (queryType.equals(Settings.MSGV_RANKED_QUERY)) { if (ps.getAttribute("isPagingRanked") != null && ps.getAttribute("isPagingRanked").equals(currentSelectorPage + "")) { ps.removeAttribute("isPagingRanked"); unrankedHits = (IngridHitsWrapper) SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_RESULT_UNRANKED); if (log.isDebugEnabled()) { log.debug("Read UNRANKED hits from CACHE !!!! unrankedHits=" + unrankedHits); } } else { // process query, create QueryDescriptor qd = QueryPreProcessor.createUnrankedQueryDescriptor(request); if (qd != null) { controller.addQuery("unranked", qd); SearchState.resetSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_UNRANKED); } } } } } // get possible changes AFTER PREPROCESSING QUERY !!! // Grouping may have changed ! String grouping = (String) SearchState.getSearchStateObject(request, Settings.PARAM_GROUPING); // fire query, post process results boolean rankedColumnHasMoreGroupedPages = true; if (controller.hasQueries()) { // fire queries HashMap results = controller.search(); // check for zero results // log the result to the resource logger Iterator it = results.keySet().iterator(); boolean noResults = true; String queryTypes = ""; while (it.hasNext()) { Object key = it.next(); if (queryTypes.length() > 0) { queryTypes = queryTypes.concat(","); } else { queryTypes = queryTypes.concat(key.toString()); } IngridHitsWrapper hits = (IngridHitsWrapper) results.get(key); if (hits != null && hits.length() > 0) { noResults = false; break; } } if (noResults) { String url = PortalConfig.getInstance().getString(PortalConfig.PORTAL_LOGGER_RESOURCE); if (url != null) { String queryString = SearchState.getSearchStateObjectAsString(request, Settings.PARAM_QUERY_STRING); HttpMethod method = null; try { url = url.concat("?code=NO_RESULTS_FOR_QUERY&q=") .concat(URLEncoder.encode(queryString, "UTF-8")).concat("&qtypes=") .concat(URLEncoder.encode(queryTypes, "UTF-8")); method = new GetMethod(url); method.setFollowRedirects(true); client.executeMethod(method); } catch (Throwable t) { if (log.isErrorEnabled()) { log.error("Cannot make connection to logger resource: ".concat(url), t); } } finally { if (method != null) { try { method.releaseConnection(); } catch (Throwable t) { if (log.isErrorEnabled()) { log.error("Cannot close connection to logger resource: ".concat(url), t); } } } } } } // post process ranked hits if exists if (results.containsKey("ranked")) { rankedHits = QueryResultPostProcessor.processRankedHits((IngridHitsWrapper) results.get("ranked"), selectedDS); SearchState.adaptSearchState(request, Settings.MSG_SEARCH_RESULT_RANKED, rankedHits); SearchState.adaptSearchState(request, Settings.MSG_SEARCH_FINISHED_RANKED, Settings.MSGV_TRUE); // GROUPING ONLY !!! if (grouping != null && !grouping.equals(IngridQuery.GROUPED_OFF)) { // get the grouping starthits history from session // create and initialize if not exists // NOTICE: when grouping by domain the navigation is like ungrouped navigation, so multiple pages are rendered ! try { ArrayList groupedStartHits = (ArrayList) SearchState.getSearchStateObject(request, Settings.PARAM_GROUPING_STARTHITS); if (groupedStartHits == null) { groupedStartHits = new ArrayList(); SearchState.adaptSearchState(request, Settings.PARAM_GROUPING_STARTHITS, groupedStartHits); } // set starthit of NEXT page ! ensure correct size of Array ! Notice: currentSelectorPage is 1 for first page ! while (currentSelectorPage >= groupedStartHits.size()) { groupedStartHits.add(new Integer(0)); } // set start hit for next page (grouping) int nextStartHit = rankedHits.getGoupedHitsLength(); groupedStartHits.set(currentSelectorPage, new Integer(nextStartHit)); // check whether there are more pages for grouped hits ! this is done due to former Bug in Backend ! // still necessary ? well, these former checks don't damage anything ... if (rankedHits.length() <= rankedHits.getGoupedHitsLength()) { // total number of hits (ungrouped) already processed -> no more pages rankedColumnHasMoreGroupedPages = false; } else { int currentStartHit = ((Integer) groupedStartHits.get(currentSelectorPage - 1)) .intValue(); if (nextStartHit == currentStartHit) { // start hit for next page same as start hit for current page -> no more pages rankedColumnHasMoreGroupedPages = false; } } } catch (Exception ex) { if (log.isInfoEnabled()) { log.info("Problems processing grouping starthits RANKED", ex); } } } } // post process unranked hits if exists if (results.containsKey("unranked")) { unrankedHits = QueryResultPostProcessor .processUnrankedHits((IngridHitsWrapper) results.get("unranked"), selectedDS); SearchState.adaptSearchState(request, Settings.MSG_SEARCH_RESULT_UNRANKED, unrankedHits); SearchState.adaptSearchState(request, Settings.MSG_SEARCH_FINISHED_UNRANKED, Settings.MSGV_TRUE); // get the grouping starthits history from session // create and initialize if not exists try { ArrayList groupedStartHits = (ArrayList) SearchState.getSearchStateObject(request, Settings.PARAM_GROUPING_STARTHITS_UNRANKED); if (groupedStartHits == null || unrankedHits == null) { groupedStartHits = new ArrayList(); groupedStartHits.add(new Integer(0)); SearchState.adaptSearchState(request, Settings.PARAM_GROUPING_STARTHITS_UNRANKED, groupedStartHits); } else { // set start hit for next page (grouping) int nextStartHit = unrankedHits.getGoupedHitsLength(); groupedStartHits.add(currentSelectorPageUnranked, new Integer(nextStartHit)); } } catch (Exception ex) { if (log.isDebugEnabled()) { log.debug("Problems processing grouping starthits UNRANKED", ex); } if (log.isInfoEnabled()) { log.info( "Problems processing grouping starthits UNRANKED. switch to debug level to get the exception logged."); } } } } int totalNumberOfRankedHits = 0; if (rankedHits != null) { totalNumberOfRankedHits = (int) rankedHits.length(); } // adapt settings of ranked page navigation HashMap rankedPageNavigation = UtilsSearch.getPageNavigation(rankedStartHit, Settings.SEARCH_RANKED_HITS_PER_PAGE, totalNumberOfRankedHits, Settings.SEARCH_RANKED_NUM_PAGES_TO_SELECT); int totalNumberOfUnrankedHits = 0; if (unrankedHits != null) { if (filter != null && filter.equals(Settings.RESULT_KEY_PLUG_ID)) { totalNumberOfUnrankedHits = (int) unrankedHits.length(); } else { totalNumberOfUnrankedHits = unrankedHits.getInVolvedPlugs(); } } // adapt settings of unranked page navigation HashMap unrankedPageNavigation = UtilsSearch.getPageNavigation(unrankedStartHit, Settings.SEARCH_UNRANKED_HITS_PER_PAGE, totalNumberOfUnrankedHits, Settings.SEARCH_UNRANKED_NUM_PAGES_TO_SELECT); Object rankedSearchFinished = SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_RANKED); Object unrankedSearchFinished = SearchState.getSearchStateObject(request, Settings.MSG_SEARCH_FINISHED_UNRANKED); /* // DON'T SHOW separate Template when no results ! This one is never displayed when JS is active and search is performed // initially (cause then always two columns are rendered (via iframes)). Only afterwards, e.g. when similar terms are // clicked, this template is displayed, causing changes of result content (when similar terms are displayed). // WE DON'T WANT TO CHANGE RESULTS CONTENT, WHEN SIMILAR TERMS ARE CKLICKED (DO we ???) if (rankedSearchFinished != null && unrankedSearchFinished != null && numberOfRankedHits == 0 && numberOfUnrankedHits == 0 && (renderOneResultColumnUnranked && renderOneResultColumnRanked) && filter.length() == 0) { // query string will be displayed when no results ! String queryString = SearchState.getSearchStateObjectAsString(request, Settings.PARAM_QUERY_STRING); context.put("queryString", queryString); setDefaultViewPage(TEMPLATE_NO_RESULT); super.doView(request, response); return; } */ // ---------------------------------- // prepare view // ---------------------------------- // GROUPING // adapt page navigation for grouping in left column if (renderResultColumnRanked) { if (grouping != null && !grouping.equals(IngridQuery.GROUPED_OFF)) { UtilsSearch.adaptRankedPageNavigationToGrouping(rankedPageNavigation, currentSelectorPage, rankedColumnHasMoreGroupedPages, totalNumberOfRankedHits, request); if (grouping.equals(IngridQuery.GROUPED_BY_PARTNER)) { context.put("grouping", "partner"); } else if (grouping.equals(IngridQuery.GROUPED_BY_ORGANISATION)) { context.put("grouping", "provider"); } else if (grouping.equals(IngridQuery.GROUPED_BY_DATASOURCE)) { context.put("grouping", "domain"); } } } // adapt page navigation for right column (always grouped) if (renderResultColumnUnranked) { unrankedPageNavigation.put(Settings.PARAM_CURRENT_SELECTOR_PAGE_UNRANKED, new Integer(currentSelectorPageUnranked)); // check if we have more results to come if (unrankedHits != null) { int groupedHitsLength = unrankedHits.getGoupedHitsLength(); if (groupedHitsLength > 0 && totalNumberOfUnrankedHits > 0) { if (totalNumberOfUnrankedHits <= groupedHitsLength) { unrankedPageNavigation.put("selectorHasNextPage", new Boolean(false)); } else { unrankedPageNavigation.put("selectorHasNextPage", new Boolean(true)); } } } } boolean showAdminContent = false; if (request.getUserPrincipal() != null) { showAdminContent = request.getUserPrincipal().getName().equals("admin"); } // check for one column rendering if (reqParam != null) { // check if we have to render only the ranked column if (reqParam.equals("true")) { request.setAttribute(GenericServletPortlet.PARAM_VIEW_PAGE, TEMPLATE_RESULT_JS_RANKED); renderResultColumnUnranked = false; } else { request.setAttribute(GenericServletPortlet.PARAM_VIEW_PAGE, TEMPLATE_RESULT_JS_UNRANKED); renderResultColumnRanked = false; } // check for js enabled iframe rendering } else if (currentView.equals(TEMPLATE_RESULT_FILTERED_ONECOLUMN)) { if (filter.equals(Settings.PARAMV_GROUPING_PLUG_ID) && rankedHits == null) { renderResultColumnRanked = false; context.put("IS_RANKED", new Boolean(false)); } else { // grouping by domain renderResultColumnUnranked = false; context.put("IS_RANKED", new Boolean(true)); } } else if (currentView.equals(TEMPLATE_RESULT_ADDRESS)) { renderResultColumnUnranked = false; context.put("ds", request.getParameter("ds")); // check for js enabled iframe rendering } else if (hasJavaScript && queryType.equals(Settings.MSGV_NEW_QUERY)) { // if javascript and new query, set template to javascript enabled iframe template // exit method!! request.setAttribute(GenericServletPortlet.PARAM_VIEW_PAGE, TEMPLATE_RESULT_JS); context.put("ds", request.getParameter("ds")); if (rankedHits != null) { if (PortalConfig.getInstance().getBoolean(PortalConfig.PORTAL_ENABLE_SEARCH_FACETE, false)) { UtilsFacete.checkForExistingFacete((IngridHitsWrapper) rankedHits, request); UtilsFacete.setParamsToContext(request, context, true); } } context.put("adminContent", showAdminContent); context.put("rankedPageSelector", rankedPageNavigation); context.put("rankedResultList", rankedHits); context.put("rankedSearchFinished", rankedSearchFinished); context.put("unrankedResultUrl", response.encodeURL(((RequestContext) request.getAttribute(RequestContext.REQUEST_PORTALENV)) .getRequest().getContextPath() + "/portal/search-result-js.psml" + SearchState.getURLParamsMainSearch(request) + "&js_ranked=false")); super.doView(request, response); return; } context.put("adminContent", showAdminContent); context.put("rankedPageSelector", rankedPageNavigation); context.put("unrankedPageSelector", unrankedPageNavigation); if (rankedHits != null) { if (PortalConfig.getInstance().getBoolean(PortalConfig.PORTAL_ENABLE_SEARCH_FACETE, false)) { UtilsFacete.checkForExistingFacete((IngridHitsWrapper) rankedHits, request); UtilsFacete.setParamsToContext(request, context, true); } } context.put("rankedResultList", rankedHits); context.put("unrankedResultList", unrankedHits); context.put("rankedSearchFinished", rankedSearchFinished); context.put("unrankedSearchFinished", unrankedSearchFinished); //context.put("providerMap", getProviderMap()); super.doView(request, response); }
From source file:com.twinsoft.convertigo.beans.connectors.HttpConnector.java
protected int doExecuteMethod(final HttpMethod method, Context context) throws ConnectionException, URIException, MalformedURLException { int statuscode = -1; // Tells the method to automatically handle authentication. method.setDoAuthentication(true);// w w w . j a v a 2 s . co m // Tells the method to automatically handle redirection. method.setFollowRedirects(false); HttpPool httpPool = ((AbstractHttpTransaction) context.transaction).getHttpPool(); HttpClient httpClient = context.getHttpClient3(httpPool); try { // Display the cookies if (handleCookie) { Cookie[] cookies = httpState.getCookies(); if (Engine.logBeans.isTraceEnabled()) Engine.logBeans.trace( "(HttpConnector) HttpClient request cookies:" + Arrays.asList(cookies).toString()); } forwardHeader(new HeaderForwarder() { public void add(String name, String value, String forwardPolicy) { if (HttpConnector.HTTP_HEADER_FORWARD_POLICY_IGNORE.equals(forwardPolicy)) { Header exHeader = method.getRequestHeader(name); if (exHeader != null) { // Nothing to do Engine.logEngine.debug("(WebViewer) Forwarding header '" + name + "' has been ignored due to forward policy"); } else { method.setRequestHeader(name, value); Engine.logEngine.debug("(WebViewer) Header forwarded and added: " + name + "=" + value); } } else if (HttpConnector.HTTP_HEADER_FORWARD_POLICY_REPLACE.equals(forwardPolicy)) { method.setRequestHeader(name, value); Engine.logEngine.debug("(WebViewer) Header forwarded and replaced: " + name + "=" + value); } else if (HttpConnector.HTTP_HEADER_FORWARD_POLICY_MERGE.equals(forwardPolicy)) { Header exHeader = method.getRequestHeader(name); if (exHeader != null) value = exHeader.getValue() + ", " + value; method.setRequestHeader(name, value); Engine.logEngine.debug("(WebViewer) Header forwarded and merged: " + name + "=" + value); } } }); // handle oAuthSignatures if any if (oAuthKey != null && oAuthSecret != null && oAuthToken != null && oAuthTokenSecret != null) { oAuthConsumer = new HttpOAuthConsumer(oAuthKey, oAuthSecret, hostConfiguration); oAuthConsumer.setTokenWithSecret(oAuthToken, oAuthTokenSecret); oAuthConsumer.sign(method); oAuthConsumer = null; } HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, httpPool); hostConfiguration.getParams().setIntParameter(HttpConnectionParams.SO_TIMEOUT, (int) context.requestedObject.getResponseTimeout() * 1000); hostConfiguration.getParams().setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT, (int) context.requestedObject.getResponseTimeout() * 1000); Engine.logBeans.debug("(HttpConnector) HttpClient: executing method..."); statuscode = httpClient.executeMethod(hostConfiguration, method, httpState); Engine.logBeans.debug("(HttpConnector) HttpClient: end of method successfull"); // Display the cookies if (handleCookie) { Cookie[] cookies = httpState.getCookies(); if (Engine.logBeans.isTraceEnabled()) Engine.logBeans.trace( "(HttpConnector) HttpClient response cookies:" + Arrays.asList(cookies).toString()); } } catch (SocketTimeoutException e) { throw new ConnectionException( "Timeout reached (" + context.requestedObject.getResponseTimeout() + " sec)"); } catch (IOException e) { if (!context.requestedObject.runningThread.bContinue) return statuscode; try { HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, httpPool); Engine.logBeans.warn("(HttpConnector) HttpClient: connection error to " + sUrl + ": " + e.getMessage() + "; retrying method"); statuscode = httpClient.executeMethod(hostConfiguration, method, httpState); Engine.logBeans.debug("(HttpConnector) HttpClient: end of method successfull"); } catch (IOException ee) { throw new ConnectionException("Connection error to " + sUrl, ee); } } catch (OAuthException eee) { throw new ConnectionException("OAuth Connection error to " + sUrl, eee); } return statuscode; }
From source file:davmail.exchange.ews.EwsExchangeSession.java
/** * Check endpoint url./* w w w . j av a 2 s . com*/ * * @param endPointUrl endpoint url * @throws IOException on error */ protected void checkEndPointUrl(String endPointUrl) throws IOException { HttpMethod getMethod = new GetMethod(endPointUrl); getMethod.setFollowRedirects(false); try { int status = DavGatewayHttpClientFacade.executeNoRedirect(httpClient, getMethod); if (status == HttpStatus.SC_UNAUTHORIZED) { throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED"); } else if (status != HttpStatus.SC_MOVED_TEMPORARILY) { throw DavGatewayHttpClientFacade.buildHttpException(getMethod); } // check Location Header locationHeader = getMethod.getResponseHeader("Location"); if (locationHeader == null || !"/ews/services.wsdl".equalsIgnoreCase(locationHeader.getValue())) { throw new IOException("Ews endpoint not available at " + getMethod.getURI().toString()); } } finally { getMethod.releaseConnection(); } }
From source file:com.zimbra.cs.zimlet.ProxyServlet.java
private void doProxy(HttpServletRequest req, HttpServletResponse resp) throws IOException { ZimbraLog.clearContext();/*w w w. ja v a2s . co m*/ boolean isAdmin = isAdminRequest(req); AuthToken authToken = isAdmin ? getAdminAuthTokenFromCookie(req, resp, true) : getAuthTokenFromCookie(req, resp, true); if (authToken == null) { String zAuthToken = req.getParameter(QP_ZAUTHTOKEN); if (zAuthToken != null) { try { authToken = AuthProvider.getAuthToken(zAuthToken); if (authToken.isExpired()) { resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken expired"); return; } if (!authToken.isRegistered()) { resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "authtoken is invalid"); return; } if (isAdmin && !authToken.isAdmin()) { resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "permission denied"); return; } } catch (AuthTokenException e) { resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "unable to parse authtoken"); return; } } } if (authToken == null) { resp.sendError(HttpServletResponse.SC_UNAUTHORIZED, "no authtoken cookie"); return; } // get the posted body before the server read and parse them. byte[] body = copyPostedData(req); // sanity check String target = req.getParameter(TARGET_PARAM); if (target == null) { resp.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } // check for permission URL url = new URL(target); if (!isAdmin && !checkPermissionOnTarget(url, authToken)) { resp.sendError(HttpServletResponse.SC_FORBIDDEN); return; } // determine whether to return the target inline or store it as an upload String uploadParam = req.getParameter(UPLOAD_PARAM); boolean asUpload = uploadParam != null && (uploadParam.equals("1") || uploadParam.equalsIgnoreCase("true")); HttpMethod method = null; try { HttpClient client = ZimbraHttpConnectionManager.getExternalHttpConnMgr().newHttpClient(); HttpProxyUtil.configureProxy(client); String reqMethod = req.getMethod(); if (reqMethod.equalsIgnoreCase("GET")) { method = new GetMethod(target); } else if (reqMethod.equalsIgnoreCase("POST")) { PostMethod post = new PostMethod(target); if (body != null) post.setRequestEntity(new ByteArrayRequestEntity(body, req.getContentType())); method = post; } else if (reqMethod.equalsIgnoreCase("PUT")) { PutMethod put = new PutMethod(target); if (body != null) put.setRequestEntity(new ByteArrayRequestEntity(body, req.getContentType())); method = put; } else if (reqMethod.equalsIgnoreCase("DELETE")) { method = new DeleteMethod(target); } else { ZimbraLog.zimlet.info("unsupported request method: " + reqMethod); resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); return; } // handle basic auth String auth, user, pass; auth = req.getParameter(AUTH_PARAM); user = req.getParameter(USER_PARAM); pass = req.getParameter(PASS_PARAM); if (auth != null && user != null && pass != null) { if (!auth.equals(AUTH_BASIC)) { ZimbraLog.zimlet.info("unsupported auth type: " + auth); resp.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } HttpState state = new HttpState(); state.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, pass)); client.setState(state); method.setDoAuthentication(true); } Enumeration headers = req.getHeaderNames(); while (headers.hasMoreElements()) { String hdr = (String) headers.nextElement(); ZimbraLog.zimlet.debug("incoming: " + hdr + ": " + req.getHeader(hdr)); if (canProxyHeader(hdr)) { ZimbraLog.zimlet.debug("outgoing: " + hdr + ": " + req.getHeader(hdr)); if (hdr.equalsIgnoreCase("x-host")) method.getParams().setVirtualHost(req.getHeader(hdr)); else method.addRequestHeader(hdr, req.getHeader(hdr)); } } try { if (!(reqMethod.equalsIgnoreCase("POST") || reqMethod.equalsIgnoreCase("PUT"))) { method.setFollowRedirects(true); } HttpClientUtil.executeMethod(client, method); } catch (HttpException ex) { ZimbraLog.zimlet.info("exception while proxying " + target, ex); resp.sendError(HttpServletResponse.SC_NOT_FOUND); return; } int status = method.getStatusLine() == null ? HttpServletResponse.SC_INTERNAL_SERVER_ERROR : method.getStatusCode(); // workaround for Alexa Thumbnails paid web service, which doesn't bother to return a content-type line Header ctHeader = method.getResponseHeader("Content-Type"); String contentType = ctHeader == null || ctHeader.getValue() == null ? DEFAULT_CTYPE : ctHeader.getValue(); InputStream targetResponseBody = method.getResponseBodyAsStream(); if (asUpload) { String filename = req.getParameter(FILENAME_PARAM); if (filename == null || filename.equals("")) filename = new ContentType(contentType).getParameter("name"); if ((filename == null || filename.equals("")) && method.getResponseHeader("Content-Disposition") != null) filename = new ContentDisposition(method.getResponseHeader("Content-Disposition").getValue()) .getParameter("filename"); if (filename == null || filename.equals("")) filename = "unknown"; List<Upload> uploads = null; if (targetResponseBody != null) { try { Upload up = FileUploadServlet.saveUpload(targetResponseBody, filename, contentType, authToken.getAccountId()); uploads = Arrays.asList(up); } catch (ServiceException e) { if (e.getCode().equals(MailServiceException.UPLOAD_REJECTED)) status = HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE; else status = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; } } resp.setStatus(status); FileUploadServlet.sendResponse(resp, status, req.getParameter(FORMAT_PARAM), null, uploads, null); } else { resp.setStatus(status); resp.setContentType(contentType); for (Header h : method.getResponseHeaders()) if (canProxyHeader(h.getName())) resp.addHeader(h.getName(), h.getValue()); if (targetResponseBody != null) ByteUtil.copy(targetResponseBody, true, resp.getOutputStream(), true); } } finally { if (method != null) method.releaseConnection(); } }
From source file:com.twinsoft.convertigo.beans.connectors.SiteClipperConnector.java
private void doProcessRequest(Shuttle shuttle) throws IOException, ServletException, EngineException { shuttle.statisticsTaskID = context.statistics.start(EngineStatistics.GET_DOCUMENT); try {//from www. j a v a 2 s . c o m shuttle.sharedScope = context.getSharedScope(); String domain = shuttle.getRequest(QueryPart.host) + shuttle.getRequest(QueryPart.port); Engine.logSiteClipper.trace("(SiteClipperConnector) Prepare the request for the domain " + domain); if (!shouldRewrite(domain)) { Engine.logSiteClipper.info( "(SiteClipperConnector) The domain " + domain + " is not allowed with this connector"); shuttle.response.sendError(HttpServletResponse.SC_FORBIDDEN, "The domain " + domain + " is not allowed with this connector"); return; } String uri = shuttle.getRequest(QueryPart.uri); Engine.logSiteClipper.info("Preparing " + shuttle.request.getMethod() + " " + shuttle.getRequestUrl()); HttpMethod httpMethod = null; XulRecorder xulRecorder = context.getXulRecorder(); if (xulRecorder != null) { httpMethod = shuttle.httpMethod = xulRecorder.getRecord(shuttle.getRequestUrlAndQuery()); } if (httpMethod == null) { try { switch (shuttle.getRequestHttpMethodType()) { case GET: httpMethod = new GetMethod(uri); break; case POST: httpMethod = new PostMethod(uri); ((PostMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case PUT: httpMethod = new PutMethod(uri); ((PutMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case DELETE: httpMethod = new DeleteMethod(uri); break; case HEAD: httpMethod = new HeadMethod(uri); break; case OPTIONS: httpMethod = new OptionsMethod(uri); break; case TRACE: httpMethod = new TraceMethod(uri); break; default: throw new ServletException( "(SiteClipperConnector) unknown http method " + shuttle.request.getMethod()); } httpMethod.setFollowRedirects(false); } catch (Exception e) { throw new ServletException( "(SiteClipperConnector) unexpected exception will building the http method : " + e.getMessage()); } shuttle.httpMethod = httpMethod; SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Request screen class: " + screenClass.getName()); for (String name : Collections .list(GenericUtils.<Enumeration<String>>cast(shuttle.request.getHeaderNames()))) { if (requestHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring request header " + name); } else { String value = shuttle.request.getHeader(name); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying request header " + name + "=" + value); shuttle.setRequestCustomHeader(name, value); } } Engine.logSiteClipper.debug("(SiteClipperConnector) applying request rules for the screenclass " + screenClass.getName()); for (IRequestRule rule : screenClass.getRequestRules()) { if (rule.isEnabled()) { Engine.logSiteClipper .trace("(SiteClipperConnector) applying request rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnRequest(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the request rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled request rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.requestCustomHeaders.entrySet()) { Engine.logSiteClipper.trace("(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); httpMethod.addRequestHeader(header.getKey(), header.getValue()); } String queryString = shuttle.request.getQueryString(); if (queryString != null) { try { // Fake test in order to check query string validity new URI("http://localhost/index?" + queryString, true, httpMethod.getParams().getUriCharset()); } catch (URIException e) { // Bugfix #2103 StringBuffer newQuery = new StringBuffer(); for (String part : RegexpUtils.pattern_and.split(queryString)) { String[] pair = RegexpUtils.pattern_equals.split(part, 2); try { newQuery.append('&') .append(URLEncoder.encode(URLDecoder.decode(pair[0], "UTF-8"), "UTF-8")); if (pair.length > 1) { newQuery.append('=').append( URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8")); } } catch (UnsupportedEncodingException ee) { Engine.logSiteClipper .trace("(SiteClipperConnector) failed to encode query part : " + part); } } queryString = newQuery.length() > 0 ? newQuery.substring(1) : newQuery.toString(); Engine.logSiteClipper.trace("(SiteClipperConnector) re-encode query : " + queryString); } } Engine.logSiteClipper.debug("(SiteClipperConnector) Copying the query string : " + queryString); httpMethod.setQueryString(queryString); // if (context.httpState == null) { // Engine.logSiteClipper.debug("(SiteClipperConnector) Creating new HttpState for context id " + context.contextID); // context.httpState = new HttpState(); // } else { // Engine.logSiteClipper.debug("(SiteClipperConnector) Using HttpState of context id " + context.contextID); // } getHttpState(shuttle); HostConfiguration hostConfiguration = getHostConfiguration(shuttle); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setBooleanParameter("http.connection.stalecheck", true); httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, true)); Engine.logSiteClipper.info("Requesting " + httpMethod.getName() + " " + hostConfiguration.getHostURL() + httpMethod.getURI().toString()); HttpClient httpClient = context.getHttpClient3(shuttle.getHttpPool()); HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, shuttle.getHttpPool()); httpClient.executeMethod(hostConfiguration, httpMethod, context.httpState); } else { Engine.logSiteClipper.info("Retrieve recorded response from Context"); } int status = httpMethod.getStatusCode(); shuttle.processState = ProcessState.response; Engine.logSiteClipper.info("Request terminated with status " + status); shuttle.response.setStatus(status); if (Engine.isStudioMode() && status == HttpServletResponse.SC_OK && shuttle.getResponseMimeType().startsWith("text/")) { fireDataChanged(new ConnectorEvent(this, shuttle.getResponseAsString())); } SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Response screen class: " + screenClass.getName()); if (Engine.isStudioMode()) { Engine.theApp.fireObjectDetected(new EngineEvent(screenClass)); } for (Header header : httpMethod.getResponseHeaders()) { String name = header.getName(); if (responseHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring response header " + name); } else { String value = header.getValue(); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying response header " + name + "=" + value); shuttle.responseCustomHeaders.put(name, value); } } Engine.logSiteClipper.debug( "(SiteClipperConnector) applying response rules for the screenclass " + screenClass.getName()); for (IResponseRule rule : screenClass.getResponseRules()) { if (rule.isEnabled()) { Engine.logSiteClipper.trace("(SiteClipperConnector) applying response rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnResponse(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the response rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled response rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.responseCustomHeaders.entrySet()) { Engine.logSiteClipper.trace( "(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); shuttle.response.addHeader(header.getKey(), header.getValue()); } if (shuttle.postInstructions != null) { JSONArray instructions = new JSONArray(); for (IClientInstruction instruction : shuttle.postInstructions) { try { instructions.put(instruction.getInstruction()); } catch (JSONException e) { Engine.logSiteClipper.error( "(SiteClipperConnector) Failed to add a post instruction due to a JSONException", e); } } String codeToInject = "<script>C8O_postInstructions = " + instructions.toString() + "</script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/jquery.min.js\"></script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/siteclipper.js\"></script>\n"; String content = shuttle.getResponseAsString(); Matcher matcher = HtmlLocation.head_top.matcher(content); String newContent = RegexpUtils.inject(matcher, codeToInject); if (newContent == null) { matcher = HtmlLocation.body_top.matcher(content); newContent = RegexpUtils.inject(matcher, codeToInject); } if (newContent != null) { shuttle.setResponseAsString(newContent); } else { Engine.logSiteClipper.info( "(SiteClipperConnector) Failed to find a head or body tag in the response content"); Engine.logSiteClipper.trace("(SiteClipperConnector) Response content : \"" + content + "\""); } } long nbBytes = 0L; if (shuttle.responseAsString != null && shuttle.responseAsString.hashCode() != shuttle.responseAsStringOriginal.hashCode()) { OutputStream os = shuttle.response.getOutputStream(); switch (shuttle.getResponseContentEncoding()) { case gzip: os = new GZIPOutputStream(os); break; case deflate: os = new DeflaterOutputStream(os, new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true)); break; default: break; } nbBytes = shuttle.responseAsByte.length; IOUtils.write(shuttle.responseAsString, os, shuttle.getResponseCharset()); os.close(); } else { InputStream is = (shuttle.responseAsByte == null) ? httpMethod.getResponseBodyAsStream() : new ByteArrayInputStream(shuttle.responseAsByte); if (is != null) { nbBytes = 0; OutputStream os = shuttle.response.getOutputStream(); int read = is.read(); while (read >= 0) { os.write(read); os.flush(); read = is.read(); nbBytes++; } is.close(); // nbBytes = IOUtils.copyLarge(is, shuttle.response.getOutputStream()); Engine.logSiteClipper .trace("(SiteClipperConnector) Response body copyied (" + nbBytes + " bytes)"); } } shuttle.response.getOutputStream().close(); shuttle.score = getScore(nbBytes); Engine.logSiteClipper .debug("(SiteClipperConnector) Request terminated with a score of " + shuttle.score); } finally { long duration = context.statistics.stop(shuttle.statisticsTaskID); if (context.requestedObject != null) { try { Engine.theApp.billingManager.insertBilling(context, Long.valueOf(duration), Long.valueOf(shuttle.score)); } catch (Exception e) { Engine.logContext.warn("Unable to insert billing ticket (the billing is thus ignored): [" + e.getClass().getName() + "] " + e.getMessage()); } } } }
From source file:com.twinsoft.convertigo.engine.servlets.ReverseProxyServlet.java
/** * Executes the {@link HttpMethod} passed in and sends the proxy response * back to the client via the given {@link HttpServletResponse} * //from w w w . jav a 2 s.c om * @param httpMethodProxyRequest * An object representing the proxy request to be made * @param httpServletResponse * An object by which we can send the proxied response back to * the client * @throws IOException * Can be thrown by the {@link HttpClient}.executeMethod * @throws ServletException * Can be thrown to indicate that another error has occurred * @throws EngineException */ private void doRequest(HttpMethodType httpMethodType, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException { try { Engine.logEngine.debug("(ReverseProxyServlet) Starting request handling"); if (Boolean.parseBoolean(EnginePropertiesManager.getProperty(PropertyName.SSL_DEBUG))) { System.setProperty("javax.net.debug", "all"); Engine.logEngine.trace("(ReverseProxyServlet) Enabling SSL debug mode"); } else { System.setProperty("javax.net.debug", ""); Engine.logEngine.debug("(ReverseProxyServlet) Disabling SSL debug mode"); } String baseUrl; String projectName; String connectorName; String contextName; String extraPath; { String requestURI = httpServletRequest.getRequestURI(); Engine.logEngine.trace("(ReverseProxyServlet) Requested URI : " + requestURI); Matcher m = reg_fields.matcher(requestURI); if (m.matches() && m.groupCount() >= 5) { baseUrl = m.group(1); projectName = m.group(2); connectorName = m.group(3); contextName = m.group(4); extraPath = m.group(5); } else { throw new MalformedURLException( "The request doesn't contains needed fields : projectName, connectorName and contextName"); } } String sessionID = httpServletRequest.getSession().getId(); Engine.logEngine.debug("(ReverseProxyServlet) baseUrl : " + baseUrl + " ; projectName : " + projectName + " ; connectorName : " + connectorName + " ; contextName : " + contextName + " ; extraPath : " + extraPath + " ; sessionID : " + sessionID); Context context = Engine.theApp.contextManager.get(null, contextName, sessionID, null, projectName, connectorName, null); Project project = Engine.theApp.databaseObjectsManager.getProjectByName(projectName); context.projectName = projectName; context.project = project; ProxyHttpConnector proxyHttpConnector = (ProxyHttpConnector) project.getConnectorByName(connectorName); context.connector = proxyHttpConnector; context.connectorName = proxyHttpConnector.getName(); HostConfiguration hostConfiguration = proxyHttpConnector.hostConfiguration; // Proxy configuration String proxyServer = Engine.theApp.proxyManager.getProxyServer(); String proxyUser = Engine.theApp.proxyManager.getProxyUser(); String proxyPassword = Engine.theApp.proxyManager.getProxyPassword(); int proxyPort = Engine.theApp.proxyManager.getProxyPort(); if (!proxyServer.equals("")) { hostConfiguration.setProxy(proxyServer, proxyPort); Engine.logEngine.debug("(ReverseProxyServlet) Using proxy: " + proxyServer + ":" + proxyPort); } else { // Remove old proxy configuration hostConfiguration.setProxyHost(null); } String targetHost = proxyHttpConnector.getServer(); Engine.logEngine.debug("(ReverseProxyServlet) Target host: " + targetHost); int targetPort = proxyHttpConnector.getPort(); Engine.logEngine.debug("(ReverseProxyServlet) Target port: " + targetPort); // Configuration SSL Engine.logEngine.debug("(ReverseProxyServlet) Https: " + proxyHttpConnector.isHttps()); CertificateManager certificateManager = proxyHttpConnector.certificateManager; boolean trustAllServerCertificates = proxyHttpConnector.isTrustAllServerCertificates(); if (proxyHttpConnector.isHttps()) { Engine.logEngine.debug("(ReverseProxyServlet) Setting up SSL properties"); certificateManager.collectStoreInformation(context); Engine.logEngine.debug( "(ReverseProxyServlet) CertificateManager has changed: " + certificateManager.hasChanged); if (certificateManager.hasChanged || (!targetHost.equalsIgnoreCase(hostConfiguration.getHost())) || (hostConfiguration.getPort() != targetPort)) { Engine.logEngine .debug("(ReverseProxyServlet) Using MySSLSocketFactory for creating the SSL socket"); Protocol myhttps = new Protocol("https", MySSLSocketFactory.getSSLSocketFactory(certificateManager.keyStore, certificateManager.keyStorePassword, certificateManager.trustStore, certificateManager.trustStorePassword, trustAllServerCertificates), targetPort); hostConfiguration.setHost(targetHost, targetPort, myhttps); } Engine.logEngine.debug("(ReverseProxyServlet) Updated host configuration for SSL purposes"); } else { hostConfiguration.setHost(targetHost, targetPort); } HttpMethod httpMethodProxyRequest; String targetPath = proxyHttpConnector.getBaseDir() + extraPath; // Handle the query string if (httpServletRequest.getQueryString() != null) { targetPath += "?" + httpServletRequest.getQueryString(); } Engine.logEngine.debug("(ReverseProxyServlet) Target path: " + targetPath); Engine.logEngine.debug("(ReverseProxyServlet) Requested method: " + httpMethodType); if (httpMethodType == HttpMethodType.GET) { // Create a GET request httpMethodProxyRequest = new GetMethod(); } else if (httpMethodType == HttpMethodType.POST) { // Create a standard POST request httpMethodProxyRequest = new PostMethod(); ((PostMethod) httpMethodProxyRequest) .setRequestEntity(new InputStreamRequestEntity(httpServletRequest.getInputStream())); } else { throw new IllegalArgumentException("Unknown HTTP method: " + httpMethodType); } String charset = httpMethodProxyRequest.getParams().getUriCharset(); URI targetURI; try { targetURI = new URI(targetPath, true, charset); } catch (URIException e) { // Bugfix #1484 String newTargetPath = ""; for (String part : targetPath.split("&")) { if (!newTargetPath.equals("")) { newTargetPath += "&"; } String[] pair = part.split("="); try { newTargetPath += URLDecoder.decode(pair[0], "UTF-8") + "=" + (pair.length > 1 ? URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8") : ""); } catch (UnsupportedEncodingException ee) { newTargetPath = targetPath; } } targetURI = new URI(newTargetPath, true, charset); } httpMethodProxyRequest.setURI(targetURI); // Tells the method to automatically handle authentication. httpMethodProxyRequest.setDoAuthentication(true); HttpState httpState = getHttpState(proxyHttpConnector, context); String basicUser = proxyHttpConnector.getAuthUser(); String basicPassword = proxyHttpConnector.getAuthPassword(); String givenBasicUser = proxyHttpConnector.getGivenAuthUser(); String givenBasicPassword = proxyHttpConnector.getGivenAuthPassword(); // Basic authentication configuration String realm = null; if (!basicUser.equals("") || (basicUser.equals("") && (givenBasicUser != null))) { String userName = ((givenBasicUser == null) ? basicUser : givenBasicUser); String userPassword = ((givenBasicPassword == null) ? basicPassword : givenBasicPassword); httpState.setCredentials(new AuthScope(targetHost, targetPort, realm), new UsernamePasswordCredentials(userName, userPassword)); Engine.logEngine.debug("(ReverseProxyServlet) Credentials: " + userName + ":******"); } // Setting basic authentication for proxy if (!proxyServer.equals("") && !proxyUser.equals("")) { httpState.setProxyCredentials(new AuthScope(proxyServer, proxyPort), new UsernamePasswordCredentials(proxyUser, proxyPassword)); Engine.logEngine.debug("(ReverseProxyServlet) Proxy credentials: " + proxyUser + ":******"); } // Forward the request headers setProxyRequestHeaders(httpServletRequest, httpMethodProxyRequest, proxyHttpConnector); // Use the CEMS HttpClient HttpClient httpClient = Engine.theApp.httpClient; httpMethodProxyRequest.setFollowRedirects(false); // Execute the request int intProxyResponseCode = httpClient.executeMethod(hostConfiguration, httpMethodProxyRequest, httpState); // Check if the proxy response is a redirect // The following code is adapted from // org.tigris.noodle.filters.CheckForRedirect // Hooray for open source software if (intProxyResponseCode >= HttpServletResponse.SC_MULTIPLE_CHOICES /* 300 */ && intProxyResponseCode < HttpServletResponse.SC_NOT_MODIFIED /* 304 */) { String stringStatusCode = Integer.toString(intProxyResponseCode); String stringLocation = httpMethodProxyRequest.getResponseHeader(STRING_LOCATION_HEADER).getValue(); if (stringLocation == null) { throw new ServletException("Received status code: " + stringStatusCode + " but no " + STRING_LOCATION_HEADER + " header was found in the response"); } // Modify the redirect to go to this proxy servlet rather that // the // proxied host String redirect = handleRedirect(stringLocation, baseUrl, proxyHttpConnector); httpServletResponse.sendRedirect(redirect); Engine.logEngine.debug("(ReverseProxyServlet) Send redirect (" + redirect + ")"); return; } else if (intProxyResponseCode == HttpServletResponse.SC_NOT_MODIFIED) { // 304 needs special handling. See: // http://www.ics.uci.edu/pub/ietf/http/rfc1945.html#Code304 // We get a 304 whenever passed an 'If-Modified-Since' // header and the data on disk has not changed; server // responds w/ a 304 saying I'm not going to send the // body because the file has not changed. httpServletResponse.setIntHeader(STRING_CONTENT_LENGTH_HEADER_NAME, 0); httpServletResponse.setStatus(HttpServletResponse.SC_NOT_MODIFIED); Engine.logEngine.debug("(ReverseProxyServlet) NOT MODIFIED (304)"); return; } // Pass the response code back to the client httpServletResponse.setStatus(intProxyResponseCode); // Pass response headers back to the client Engine.logEngine.debug("(ReverseProxyServlet) Response headers back to the client:"); Header[] headerArrayResponse = httpMethodProxyRequest.getResponseHeaders(); for (Header header : headerArrayResponse) { String headerName = header.getName(); String headerValue = header.getValue(); if (!headerName.equalsIgnoreCase("Transfer-Encoding") && !headerName.equalsIgnoreCase("Set-Cookie")) { httpServletResponse.setHeader(headerName, headerValue); Engine.logEngine.debug(" " + headerName + "=" + headerValue); } } String contentType = null; Header[] contentTypeHeaders = httpMethodProxyRequest.getResponseHeaders("Content-Type"); for (Header contentTypeHeader : contentTypeHeaders) { contentType = contentTypeHeader.getValue(); break; } String pageCharset = "UTF-8"; if (contentType != null) { int iCharset = contentType.indexOf("charset="); if (iCharset != -1) { pageCharset = contentType.substring(iCharset + "charset=".length()).trim(); } Engine.logEngine.debug("(ReverseProxyServlet) Using charset: " + pageCharset); } InputStream siteIn = httpMethodProxyRequest.getResponseBodyAsStream(); // Handle gzipped content Header[] contentEncodingHeaders = httpMethodProxyRequest.getResponseHeaders("Content-Encoding"); boolean bGZip = false, bDeflate = false; for (Header contentEncodingHeader : contentEncodingHeaders) { HeaderElement[] els = contentEncodingHeader.getElements(); for (int j = 0; j < els.length; j++) { if ("gzip".equals(els[j].getName())) { Engine.logBeans.debug("(ReverseProxyServlet) Decode GZip stream"); siteIn = new GZIPInputStream(siteIn); bGZip = true; } else if ("deflate".equals(els[j].getName())) { Engine.logBeans.debug("(ReverseProxyServlet) Decode Deflate stream"); siteIn = new InflaterInputStream(siteIn, new Inflater(true)); bDeflate = true; } } } byte[] bytesDataResult; ByteArrayOutputStream baos = new ByteArrayOutputStream(2048); // String resourceUrl = projectName + targetPath; String t = context.statistics.start(EngineStatistics.APPLY_USER_REQUEST); try { // Read either from the cache, either from the remote server // InputStream is = proxyCacheManager.getResource(resourceUrl); // if (is != null) { // Engine.logEngine.debug("(ReverseProxyServlet) Getting data from cache"); // siteIn = is; // } int c = siteIn.read(); while (c > -1) { baos.write(c); c = siteIn.read(); } // if (is != null) is.close(); } finally { context.statistics.stop(t, true); } bytesDataResult = baos.toByteArray(); baos.close(); Engine.logEngine.debug("(ReverseProxyServlet) Data retrieved!"); // if (isDynamicContent(httpServletRequest.getPathInfo(), // proxyHttpConnector.getDynamicContentFiles())) { Engine.logEngine.debug("(ReverseProxyServlet) Dynamic content"); bytesDataResult = handleStringReplacements(baseUrl, contentType, pageCharset, proxyHttpConnector, bytesDataResult); String billingClassName = context.getConnector().getBillingClassName(); if (billingClassName != null) { try { Engine.logContext.debug("Billing class name required: " + billingClassName); AbstractBiller biller = (AbstractBiller) Class.forName(billingClassName).newInstance(); Engine.logContext.debug("Executing the biller"); biller.insertBilling(context); } catch (Throwable e) { Engine.logContext.warn("Unable to execute the biller (the billing is thus ignored): [" + e.getClass().getName() + "] " + e.getMessage()); } } // } // else { // Engine.logEngine.debug("(ReverseProxyServlet) Static content: " + // contentType); // // // Determine if the resource has already been cached or not // CacheEntry cacheEntry = // proxyCacheManager.getCacheEntry(resourceUrl); // if (cacheEntry instanceof FileCacheEntry) { // FileCacheEntry fileCacheEntry = (FileCacheEntry) cacheEntry; // File file = new File(fileCacheEntry.fileName); // if (!file.exists()) // proxyCacheManager.removeCacheEntry(cacheEntry); // cacheEntry = null; // } // if (cacheEntry == null) { // bytesDataResult = handleStringReplacements(contentType, // proxyHttpConnector, bytesDataResult); // // if (intProxyResponseCode == 200) { // Engine.logEngine.debug("(ReverseProxyServlet) Resource stored: " // + resourceUrl); // cacheEntry = proxyCacheManager.storeResponse(resourceUrl, // bytesDataResult); // cacheEntry.contentLength = bytesDataResult.length; // cacheEntry.contentType = contentType; // Engine.logEngine.debug("(ReverseProxyServlet) Cache entry: " + // cacheEntry); // } // } // } // Send the content to the client if (Engine.logEngine.isDebugEnabled() && MimeType.Html.is(contentType)) { Engine.logEngine.debug("Data proxied:\n" + new String(bytesDataResult, pageCharset)); } if (bGZip || bDeflate) { baos = new ByteArrayOutputStream(); OutputStream compressedOutputStream = bGZip ? new GZIPOutputStream(baos) : new DeflaterOutputStream(baos, new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true)); compressedOutputStream.write(bytesDataResult); compressedOutputStream.close(); bytesDataResult = baos.toByteArray(); baos.close(); } httpServletResponse.setContentLength(bytesDataResult.length); OutputStream outputStreamClientResponse = httpServletResponse.getOutputStream(); outputStreamClientResponse.write(bytesDataResult); Engine.logEngine.debug("(ReverseProxyServlet) End of document retransmission"); } catch (Exception e) { Engine.logEngine.error("Error while trying to proxy page", e); throw new ServletException("Error while trying to proxy page", e); } }
From source file:com.sun.jersey.client.apache.DefaultApacheHttpMethodExecutor.java
@Override public void executeMethod(final HttpMethod method, final ClientRequest cr) { final Map<String, Object> props = cr.getProperties(); method.setDoAuthentication(true);/* w w w . j a va 2s. com*/ final HttpMethodParams methodParams = method.getParams(); // Set the handle cookies property if (!cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_HANDLE_COOKIES)) { methodParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); } // Set the interactive and credential provider properties if (cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_INTERACTIVE)) { CredentialsProvider provider = (CredentialsProvider) props .get(ApacheHttpClientConfig.PROPERTY_CREDENTIALS_PROVIDER); if (provider == null) { provider = DEFAULT_CREDENTIALS_PROVIDER; } methodParams.setParameter(CredentialsProvider.PROVIDER, provider); } else { methodParams.setParameter(CredentialsProvider.PROVIDER, null); } // Set the read timeout final Integer readTimeout = (Integer) props.get(ApacheHttpClientConfig.PROPERTY_READ_TIMEOUT); if (readTimeout != null) { methodParams.setSoTimeout(readTimeout); } if (method instanceof EntityEnclosingMethod) { final EntityEnclosingMethod entMethod = (EntityEnclosingMethod) method; if (cr.getEntity() != null) { final RequestEntityWriter re = getRequestEntityWriter(cr); final Integer chunkedEncodingSize = (Integer) props .get(ApacheHttpClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE); if (chunkedEncodingSize != null) { // There doesn't seems to be a way to set the chunk size. entMethod.setContentChunked(true); // It is not possible for a MessageBodyWriter to modify // the set of headers before writing out any bytes to // the OutputStream // This makes it impossible to use the multipart // writer that modifies the content type to add a boundary // parameter writeOutBoundHeaders(cr.getHeaders(), method); // Do not buffer the request entity when chunked encoding is // set entMethod.setRequestEntity(new RequestEntity() { @Override public boolean isRepeatable() { return false; } @Override public void writeRequest(OutputStream out) throws IOException { re.writeRequestEntity(out); } @Override public long getContentLength() { return re.getSize(); } @Override public String getContentType() { return re.getMediaType().toString(); } }); } else { entMethod.setContentChunked(false); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { re.writeRequestEntity(new CommittingOutputStream(baos) { @Override protected void commit() throws IOException { writeOutBoundHeaders(cr.getMetadata(), method); } }); } catch (IOException ex) { throw new ClientHandlerException(ex); } final byte[] content = baos.toByteArray(); entMethod.setRequestEntity(new RequestEntity() { @Override public boolean isRepeatable() { return true; } @Override public void writeRequest(OutputStream out) throws IOException { out.write(content); } @Override public long getContentLength() { return content.length; } @Override public String getContentType() { return re.getMediaType().toString(); } }); } } else { writeOutBoundHeaders(cr.getHeaders(), method); } } else { writeOutBoundHeaders(cr.getHeaders(), method); // Follow redirects method.setFollowRedirects(cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_FOLLOW_REDIRECTS)); } try { httpClient.executeMethod(getHostConfiguration(httpClient, props), method, getHttpState(props)); } catch (Exception e) { method.releaseConnection(); throw new ClientHandlerException(e); } }
From source file:net.sourceforge.squirrel_sql.fw.util.IOUtilitiesImpl.java
/** * @see net.sourceforge.squirrel_sql.fw.util.IOUtilities# downloadHttpFile(java.lang.String, int, * java.lang.String, net.sourceforge.squirrel_sql.fw.util.FileWrapper) *//*w w w .j a v a2s .c om*/ public int downloadHttpFile(URL url, FileWrapper destFile, IProxySettings proxySettings) throws IOException { BufferedInputStream is = null; HttpMethod method = null; int resultCode = -1; int result = -1; try { if (s_log.isDebugEnabled()) { s_log.debug("downloadHttpFile: downloading file (" + destFile.getName() + ") from url: " + url); } HttpClient client = new HttpClient(); setupProxy(proxySettings, client, url); method = new GetMethod(url.toString()); method.setFollowRedirects(true); resultCode = client.executeMethod(method); if (s_log.isDebugEnabled()) { s_log.debug("downloadHttpFile: response code was: " + resultCode); } if (resultCode != 200) { throw new FileNotFoundException( "Failed to download file from url (" + url + "): HTTP Response Code=" + resultCode); } InputStream mis = method.getResponseBodyAsStream(); is = new BufferedInputStream(mis); if (s_log.isDebugEnabled()) { s_log.debug("downloadHttpFile: writing http response body to file: " + destFile.getAbsolutePath()); } result = copyBytesToFile(mis, destFile); } catch (IOException e) { s_log.error("downloadHttpFile: Unexpected exception while " + "attempting to open an HTTP connection to url (" + url + ") to download a file (" + destFile.getAbsolutePath() + "): " + e.getMessage(), e); throw e; } finally { closeInputStream(is); method.releaseConnection(); } return result; }