Example usage for org.apache.http.client.methods RequestBuilder get

List of usage examples for org.apache.http.client.methods RequestBuilder get

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.client.methods.*
  5. RequestBuilder
  6. get

Introduction

In this page you can find the example usage for org.apache.http.client.methods RequestBuilder get.

Prototype

public static RequestBuilder get()

Source Link

Usage

From source file:cn.wanghaomiao.seimi.http.hc.HcRequestGenerator.java

public static RequestBuilder getHttpRequestBuilder(Request request, CrawlerModel crawlerModel) {
    RequestBuilder requestBuilder;/*  ww w . jav a2 s  .com*/
    BaseSeimiCrawler crawler = crawlerModel.getInstance();
    if (request.isUseSeimiAgent()) {
        if (StringUtils.isBlank(crawler.seimiAgentHost())) {
            throw new SeimiProcessExcepiton("SeimiAgentHost is blank.");
        }
        String seimiAgentUrl = "http://" + crawler.seimiAgentHost()
                + (crawler.seimiAgentPort() != 80 ? (":" + crawler.seimiAgentPort()) : "") + "/doload";
        requestBuilder = RequestBuilder.post().setUri(seimiAgentUrl);
        requestBuilder.addParameter("url", request.getUrl());
        if (StringUtils.isNotBlank(crawler.proxy())) {
            requestBuilder.addParameter("proxy", crawler.proxy());
        }
        if (request.getSeimiAgentRenderTime() > 0) {
            requestBuilder.addParameter("renderTime", String.valueOf(request.getSeimiAgentRenderTime()));
        }
        if (StringUtils.isNotBlank(request.getSeimiAgentScript())) {
            requestBuilder.addParameter("script", request.getSeimiAgentScript());
        }
        //SeimiAgent?cookie
        if ((request.isSeimiAgentUseCookie() == null && crawlerModel.isUseCookie())
                || (request.isSeimiAgentUseCookie() != null && request.isSeimiAgentUseCookie())) {
            requestBuilder.addParameter("useCookie", "1");
        }
        if (request.getParams() != null && request.getParams().size() > 0) {
            requestBuilder.addParameter("postParam", JSON.toJSONString(request.getParams()));
        }
        if (request.getSeimiAgentContentType().val() > SeimiAgentContentType.HTML.val()) {
            requestBuilder.addParameter("contentType", request.getSeimiAgentContentType().typeVal());
        }
    } else {
        if (HttpMethod.POST.equals(request.getHttpMethod())) {
            requestBuilder = RequestBuilder.post().setUri(request.getUrl());
        } else {
            requestBuilder = RequestBuilder.get().setUri(request.getUrl());
        }
        RequestConfig config = RequestConfig.custom().setProxy(crawlerModel.getProxy())
                .setCircularRedirectsAllowed(true).build();

        if (request.getParams() != null) {
            for (Map.Entry<String, String> entry : request.getParams().entrySet()) {
                requestBuilder.addParameter(entry.getKey(), entry.getValue());
            }
        }
        requestBuilder.setConfig(config).setHeader("User-Agent",
                crawlerModel.isUseCookie() ? crawlerModel.getCurrentUA() : crawler.getUserAgent());
        requestBuilder.setHeader("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        requestBuilder.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6");
    }
    if (!CollectionUtils.isEmpty(request.getHeader())) {
        for (Map.Entry<String, String> entry : request.getHeader().entrySet()) {
            requestBuilder.setHeader(entry.getKey(), entry.getValue());
        }
    }
    return requestBuilder;
}

From source file:org.dasein.cloud.azurepack.platform.AzurePackDatabaseRequests.java

private RequestBuilder getListRequestBuilder(String uri) {
    RequestBuilder requestBuilder = RequestBuilder.get();
    addCommonHeaders(requestBuilder);//ww w  . ja v  a 2  s. c o m
    requestBuilder.setUri(String.format(uri, this.provider.getContext().getCloud().getEndpoint(),
            this.provider.getContext().getAccountNumber()));
    return requestBuilder;
}

From source file:org.daybreak.coccinella.webmagic.ImageDownloader.java

@Override
public Page download(Request request, Task task) {
    Site site = null;//  www.j  a  v  a2s  . c o  m
    if (task != null) {
        site = task.getSite();
    }
    Set<Integer> acceptStatCode;
    String charset = null;
    Map<String, String> headers = null;
    if (site != null) {
        acceptStatCode = site.getAcceptStatCode();
        charset = site.getCharset();
        headers = site.getHeaders();
    } else {
        acceptStatCode = Sets.newHashSet(200);
    }
    logger.info("downloading image " + request.getUrl());
    RequestBuilder requestBuilder;
    if (request instanceof CrawlerRequest) {
        CrawlerRequest crawlerRequest = (CrawlerRequest) request;
        try {
            requestBuilder = RequestBuilder.post().setUri(crawlerRequest.getUrl())
                    .setEntity(crawlerRequest.createEntity());
        } catch (UnsupportedEncodingException ex) {
            logger.warn("The encoding is not supported: " + crawlerRequest.getCrawler().getEncode());
            return null;
        }
    } else {
        requestBuilder = RequestBuilder.get().setUri(request.getUrl());
    }
    if (headers != null) {
        for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
            requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
        }
    }
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
            .setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut())
            .setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.BEST_MATCH);
    if (site != null && site.getHttpProxy() != null) {
        requestConfigBuilder.setProxy(site.getHttpProxy());
    }
    requestBuilder.setConfig(requestConfigBuilder.build());
    CloseableHttpResponse httpResponse = null;
    try {
        httpResponse = getHttpClient(site).execute(requestBuilder.build());
        int statusCode = httpResponse.getStatusLine().getStatusCode();
        if (acceptStatCode.contains(statusCode)) {
            ImagePage imagePage = new ImagePage(ImageIO.read(httpResponse.getEntity().getContent()));
            imagePage.setRawText("");
            imagePage.setUrl(new PlainText(request.getUrl()));
            imagePage.setRequest(request);
            imagePage.setStatusCode(httpResponse.getStatusLine().getStatusCode());
            return imagePage;
        } else {
            logger.warn("code error " + statusCode + "\t" + request.getUrl());
            return null;
        }
    } catch (IOException e) {
        logger.warn("download image " + request.getUrl() + " error", e);
        if (site.getCycleRetryTimes() > 0) {
            return addToCycleRetry(request, site);
        }
        return null;
    } finally {
        try {
            if (httpResponse != null) {
                //ensure the connection is released back to pool
                EntityUtils.consume(httpResponse.getEntity());
            }
        } catch (IOException e) {
            logger.warn("close response fail", e);
        }
    }
}

From source file:org.jmonkey.external.bintray.BintrayApiClient.java

public List<BintrayFile> search(final String query) throws IOException {

    try (CloseableHttpClient httpClient = this.createClient()) {

        HttpUriRequest request = RequestBuilder.get().setUri("https://api.bintray.com/search/packages/maven?q=*"
                + cleanQuery(query) + "*&owner=" + config.getSubject() + "&repo=" + config.getRepo()).build();

        try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {

            int statusCode = httpResponse.getStatusLine().getStatusCode();

            if (statusCode >= 400) {
                // @TODO: provide a consumable error
            }//  w ww  .j av  a2s .  c  o m

            HttpEntity entity = httpResponse.getEntity();
            String entityString = EntityUtils.toString(entity);
            EntityUtils.consume(entity);

            List<BintrayFile> results = JmeResourceWebsite.getObjectMapper().readValue(entityString,
                    new TypeReference<List<BintrayFile>>() {
                    });

            return results;
        }
    }
}

From source file:com.angelmmg90.consumerservicespotify.configuration.SpringWebConfig.java

@Bean
public static RestTemplate getTemplate() throws IOException {
    if (template == null) {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(PROXY_HOST, PROXY_PORT),
                new UsernamePasswordCredentials(PROXY_USER, PROXY_PASSWORD));

        Header[] h = new Header[3];
        h[0] = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json");
        h[1] = new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + ACCESS_TOKEN);

        List<Header> headers = new ArrayList<>(Arrays.asList(h));

        HttpClientBuilder clientBuilder = HttpClientBuilder.create();

        clientBuilder.useSystemProperties();
        clientBuilder.setProxy(new HttpHost(PROXY_HOST, PROXY_PORT));
        clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        clientBuilder.setDefaultHeaders(headers).build();
        String SAMPLE_URL = "https://api.spotify.com/v1/users/yourUserName/playlists/7HHFd1tNiIFIwYwva5MTNv";

        HttpUriRequest request = RequestBuilder.get().setUri(SAMPLE_URL).build();

        clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());

        CloseableHttpClient client = clientBuilder.build();
        client.execute(request);//from  ww  w.j a v  a  2s . c  om

        HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory();
        factory.setHttpClient(client);

        template = new RestTemplate();
        template.setRequestFactory(factory);
    }

    return template;
}

From source file:org.daybreak.coccinella.webmagic.CrawlerDownloader.java

@Override
public Page download(Request request, Task task) {
    Site site = null;/*from  ww w  .j  ava2  s  .  c  o m*/
    if (task != null) {
        site = task.getSite();
    }
    Set<Integer> acceptStatCode;
    String charset = null;
    Map<String, String> headers = null;
    if (site != null) {
        acceptStatCode = site.getAcceptStatCode();
        charset = site.getCharset();
        headers = site.getHeaders();
    } else {
        acceptStatCode = Sets.newHashSet(200);
    }
    logger.info("downloading page " + request.getUrl());
    RequestBuilder requestBuilder = null;
    if (request instanceof CrawlerRequest) {
        CrawlerRequest crawlerRequest = (CrawlerRequest) request;
        if (StringUtils.isNotBlank(crawlerRequest.getCrawler().getReferer())) {
            site.addHeader(HttpHeaders.REFERER, crawlerRequest.getCrawler().getReferer());
        }
        if (crawlerRequest.getCrawler().getMethod() == HttpMethod.GET) {
            requestBuilder = RequestBuilder.get().setUri(request.getUrl());
        } else if (crawlerRequest.getCrawler().getMethod() == HttpMethod.POST) {
            try {
                requestBuilder = RequestBuilder.post().setUri(crawlerRequest.getUrl())
                        .setEntity(crawlerRequest.createEntity());
            } catch (UnsupportedEncodingException ex) {
                logger.warn("The encoding is not supported: " + crawlerRequest.getCrawler().getEncode());
                return null;
            }
        }
    }

    if (requestBuilder == null) {
        return null;
    }

    if (headers != null) {
        for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
            requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
        }
    }
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
            .setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut())
            .setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.BEST_MATCH);
    if (site != null && site.getHttpProxy() != null) {
        requestConfigBuilder.setProxy(site.getHttpProxy());
    }
    requestBuilder.setConfig(requestConfigBuilder.build());
    CloseableHttpResponse httpResponse = null;
    try {
        httpResponse = getHttpClient(site).execute(requestBuilder.build());
        int statusCode = httpResponse.getStatusLine().getStatusCode();
        if (acceptStatCode.contains(statusCode)) {
            // ??
            Header[] resHeaders = httpResponse.getAllHeaders();
            for (int i = 0; i < resHeaders.length; i++) {
                if (resHeaders[i].getName().equals("Set-Cookie")) {
                    String cookie = resHeaders[i].getValue();
                    String cookieName = cookie.split("=")[0];
                    String cookieValue = cookie.split("=")[1].split(";")[0];
                    site.addCookie(cookieName, cookieValue);
                }
            }

            //charset
            if (charset == null) {
                String value = httpResponse.getEntity().getContentType().getValue();
                charset = UrlUtils.getCharset(value);
            }
            return handleResponse(request, charset, httpResponse, task);
        } else {
            logger.warn("code error " + statusCode + "\t" + request.getUrl());
            return null;
        }
    } catch (IOException e) {
        logger.warn("download page " + request.getUrl() + " error", e);
        if (site.getCycleRetryTimes() > 0) {
            return addToCycleRetry(request, site);
        }
        return null;
    } finally {
        try {
            if (httpResponse != null) {
                //ensure the connection is released back to pool
                EntityUtils.consume(httpResponse.getEntity());
            }
        } catch (IOException e) {
            logger.warn("close response fail", e);
        }
    }
}

From source file:org.eclipse.vorto.repository.RestClient.java

@SuppressWarnings("restriction")
public Attachment executeGetAttachment(String query) throws ClientProtocolException, IOException {

    CloseableHttpClient client = HttpClients.custom().build();

    HttpUriRequest request = RequestBuilder.get().setConfig(createProxyConfiguration())
            .setUri(createQuery(query)).build();
    return client.execute(request, new ResponseHandler<Attachment>() {

        @Override/*from  ww w. j  a va2s  . c  o  m*/
        public Attachment handleResponse(HttpResponse response) throws ClientProtocolException, IOException {
            String content_disposition = response.getFirstHeader("Content-Disposition").getValue();
            String filename = content_disposition
                    .substring(content_disposition.indexOf("filename = ") + "filename = ".length());
            long length = response.getEntity().getContentLength();
            String type = response.getEntity().getContentType().toString();
            byte[] content = IOUtils.toByteArray(response.getEntity().getContent());
            return new Attachment(filename, length, type, content);
        }
    });
}

From source file:com.nexmo.client.voice.endpoints.ListCallsMethod.java

@Override
public RequestBuilder makeRequest(CallsFilter filter)
        throws NexmoClientException, UnsupportedEncodingException {
    URIBuilder uriBuilder;/* w  w  w .jav  a  2 s .  com*/
    try {
        uriBuilder = new URIBuilder(this.uri);
    } catch (URISyntaxException e) {
        throw new NexmoUnexpectedException("Could not parse URI: " + this.uri);
    }
    if (filter != null) {
        List<NameValuePair> params = filter.toUrlParams();
        for (NameValuePair param : params) {
            uriBuilder.setParameter(param.getName(), param.getValue());
        }
    }
    return RequestBuilder.get().setUri(uriBuilder.toString());
}

From source file:edu.mit.scratch.ScratchStatistics.java

protected static JSONObject getStatisticsJSONObject() throws ScratchStatisticalException {
    try {//from  ww  w  .  j  ava 2 s . c  o  m
        final RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.DEFAULT).build();

        final CookieStore cookieStore = new BasicCookieStore();
        final BasicClientCookie lang = new BasicClientCookie("scratchlanguage", "en");
        final BasicClientCookie debug = new BasicClientCookie("DEBUG", "true");
        debug.setDomain(".scratch.mit.edu");
        debug.setPath("/");
        lang.setDomain(".scratch.mit.edu");
        lang.setPath("/");
        cookieStore.addCookie(debug);
        cookieStore.addCookie(lang);

        final CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(globalConfig)
                .setUserAgent(Scratch.USER_AGENT).setDefaultCookieStore(cookieStore).build();
        CloseableHttpResponse resp;

        final HttpUriRequest update = RequestBuilder.get()
                .setUri("https://scratch.mit.edu/statistics/data/daily/")
                .addHeader("Accept",
                        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
                .addHeader("Referer", "https://scratch.mit.edu").addHeader("Origin", "https://scratch.mit.edu")
                .addHeader("Accept-Encoding", "gzip, deflate, sdch")
                .addHeader("Accept-Language", "en-US,en;q=0.8").addHeader("Content-Type", "application/json")
                .addHeader("X-Requested-With", "XMLHttpRequest").build();
        try {
            resp = httpClient.execute(update);
        } catch (final IOException e) {
            e.printStackTrace();
            throw new ScratchStatisticalException();
        }

        BufferedReader rd;
        try {
            rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
        } catch (UnsupportedOperationException | IOException e) {
            e.printStackTrace();
            throw new ScratchStatisticalException();
        }

        final StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null)
            result.append(line);
        return new JSONObject(result.toString().trim());
    } catch (final UnsupportedEncodingException e) {
        e.printStackTrace();
        throw new ScratchStatisticalException();
    } catch (final Exception e) {
        e.printStackTrace();
        throw new ScratchStatisticalException();
    }
}

From source file:edu.mit.scratch.Scratch.java

public static ScratchSession createSession(final String username, String password)
        throws ScratchLoginException {
    try {/*from   ww  w . j ava 2 s.  co  m*/
        final RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.DEFAULT) // Changed due to deprecation
                .build();

        final CookieStore cookieStore = new BasicCookieStore();
        final BasicClientCookie lang = new BasicClientCookie("scratchlanguage", "en");
        lang.setDomain(".scratch.mit.edu");
        lang.setPath("/");
        cookieStore.addCookie(lang);

        final CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(globalConfig)
                .setUserAgent(Scratch.USER_AGENT).setDefaultCookieStore(cookieStore).build();
        CloseableHttpResponse resp;

        final HttpUriRequest csrf = RequestBuilder.get().setUri("https://scratch.mit.edu/csrf_token/")
                .addHeader("Accept", "*/*").addHeader("Referer", "https://scratch.mit.edu")
                .addHeader("X-Requested-With", "XMLHttpRequest").build();
        resp = httpClient.execute(csrf);
        resp.close();

        String csrfToken = null;
        for (final Cookie c : cookieStore.getCookies())
            if (c.getName().equals("scratchcsrftoken"))
                csrfToken = c.getValue();

        final JSONObject loginObj = new JSONObject();
        loginObj.put("username", username);
        loginObj.put("password", password);
        loginObj.put("captcha_challenge", "");
        loginObj.put("captcha_response", "");
        loginObj.put("embed_captcha", false);
        loginObj.put("timezone", "America/New_York");
        loginObj.put("csrfmiddlewaretoken", csrfToken);
        final HttpUriRequest login = RequestBuilder.post().setUri("https://scratch.mit.edu/accounts/login/")
                .addHeader("Accept", "application/json, text/javascript, */*; q=0.01")
                .addHeader("Referer", "https://scratch.mit.edu").addHeader("Origin", "https://scratch.mit.edu")
                .addHeader("Accept-Encoding", "gzip, deflate").addHeader("Accept-Language", "en-US,en;q=0.8")
                .addHeader("Content-Type", "application/json").addHeader("X-Requested-With", "XMLHttpRequest")
                .addHeader("X-CSRFToken", csrfToken).setEntity(new StringEntity(loginObj.toString())).build();
        resp = httpClient.execute(login);
        password = null;
        final BufferedReader rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));

        final StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null)
            result.append(line);
        final JSONObject jsonOBJ = new JSONObject(
                result.toString().substring(1, result.toString().length() - 1));
        if ((int) jsonOBJ.get("success") != 1)
            throw new ScratchLoginException();
        String ssi = null;
        String sct = null;
        String e = null;
        final Header[] headers = resp.getAllHeaders();
        for (final Header header : headers)
            if (header.getName().equals("Set-Cookie")) {
                final String value = header.getValue();
                final String[] split = value.split(Pattern.quote("; "));
                for (final String s : split) {
                    if (s.contains("=")) {
                        final String[] split2 = s.split(Pattern.quote("="));
                        final String key = split2[0];
                        final String val = split2[1];
                        if (key.equals("scratchsessionsid"))
                            ssi = val;
                        else if (key.equals("scratchcsrftoken"))
                            sct = val;
                        else if (key.equals("expires"))
                            e = val;
                    }
                }
            }
        resp.close();

        return new ScratchSession(ssi, sct, e, username);
    } catch (final IOException e) {
        e.printStackTrace();
        throw new ScratchLoginException();
    }
}