Example usage for org.apache.http.client.utils URIBuilder setUserInfo

Introduction

In this page you can find the example usage for org.apache.http.client.utils URIBuilder setUserInfo.

Prototype

public URIBuilder setUserInfo(final String userInfo) 

Source Link

Document

Sets URI user info.

Usage

From source file:org.cloudcrawler.domain.crawler.robotstxt.RobotsTxtService.java

/**
 * This method is used to evaluate if the provided uri is
 * allowed to be crawled against the robots.txt of the website.
 *
 * @param uri//ww  w.  jav  a  2  s .  c  o  m
 * @return
 * @throws Exception
 */
public boolean isAllowedUri(URI uri) throws Exception {
    URIBuilder uriBuilder = new URIBuilder();
    uriBuilder.setScheme(uri.getScheme());
    uriBuilder.setHost(uri.getHost());
    uriBuilder.setUserInfo(uri.getUserInfo());
    uriBuilder.setPath("/robots.txt");

    URI robotsTxtUri = uriBuilder.build();
    BaseRobotRules rules = (BaseRobotRules) cache.get(robotsTxtUri.toString());

    if (rules == null) {
        HttpResponse response = httpService.get(robotsTxtUri);

        try {

            // HACK! DANGER! Some sites will redirect the request to the top-level domain
            // page, without returning a 404. So look for a response which has a redirect,
            // and the fetched content is not plain text, and assume it's one of these...
            // which is the same as not having a robotstxt.txt file.

            String contentType = response.getEntity().getContentType().getValue();
            boolean isPlainText = (contentType != null) && (contentType.startsWith("text/plain"));

            if (response.getStatusLine().getStatusCode() == 404 || !isPlainText) {
                rules = robotsTxtParser.failedFetch(HttpStatus.SC_GONE);
            } else {
                StringWriter writer = new StringWriter();

                IOUtils.copy(response.getEntity().getContent(), writer);

                rules = robotsTxtParser.parseContent(uri.toString(), writer.toString().getBytes(),
                        response.getEntity().getContentType().getValue(), httpService.getUserAgent());

            }
        } catch (Exception e) {
            EntityUtils.consume(response.getEntity());
            throw e;
        }

        EntityUtils.consume(response.getEntity());
        cache.set(robotsTxtUri.toString(), 60 * 60 * 24, rules);
    }

    return rules.isAllowed(uri.toString());
}

---

From source file:com.griddynamics.jagger.invoker.http.ApacheHttpInvoker.java

@Override
protected HttpRequestBase getHttpMethod(HttpRequestBase query, String endpoint) {
    try {//  w w  w . j av a  2 s  .co m
        if (query.getURI() == null) {
            query.setURI(URI.create(endpoint));
            return query;
        } else {
            URIBuilder uriBuilder = new URIBuilder(URI.create(endpoint));
            uriBuilder.setQuery(query.getURI().getQuery());
            uriBuilder.setFragment(query.getURI().getFragment());
            uriBuilder.setUserInfo(query.getURI().getUserInfo());
            if (!query.getURI().getPath().isEmpty()) {
                uriBuilder.setPath(query.getURI().getPath());
            }
            query.setURI(uriBuilder.build());
            return query;
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.obiba.magma.datasource.mongodb.MongoDBDatasourceFactory.java

public URI getUri() {
    try {//from   ww  w .  ja  v  a  2  s. co  m
        URIBuilder uriBuilder = new URIBuilder(url);
        if (!Strings.isNullOrEmpty(username)) {
            if (Strings.isNullOrEmpty(password)) {
                uriBuilder.setUserInfo(username);
            } else {
                uriBuilder.setUserInfo(username, password);
            }
        }
        Properties prop = readOptions();
        for (Map.Entry<Object, Object> entry : prop.entrySet()) {
            uriBuilder.addParameter(entry.getKey().toString(), entry.getValue().toString());
        }
        return uriBuilder.build();
    } catch (URISyntaxException e) {
        throw new RuntimeException("Cannot create MongoDB URI", e);
    }
}

---

From source file:org.mobicents.servlet.restcomm.http.client.HttpRequestDescriptor.java

private URI base(final URI uri) {
    try {/*from  www  .j  a v a 2s .c  o m*/
        URIBuilder uriBuilder = new URIBuilder();
        uriBuilder.setScheme(uri.getScheme());
        uriBuilder.setHost(uri.getHost());
        uriBuilder.setPort(uri.getPort());
        uriBuilder.setPath(uri.getPath());

        if (uri.getUserInfo() != null) {
            uriBuilder.setUserInfo(uri.getUserInfo());
        }
        return uriBuilder.build();
    } catch (final URISyntaxException ignored) {
        // This should never happen since we are using a valid URI to construct ours.
        return null;
    }
}

---

From source file:com.cisco.oss.foundation.http.apache.ApacheHttpClient.java

private URI buildUri(HttpRequest request, Joiner joiner) {
    URI requestUri = request.getUri();

    Map<String, Collection<String>> queryParams = request.getQueryParams();
    if (queryParams != null && !queryParams.isEmpty()) {
        URIBuilder uriBuilder = new URIBuilder();
        StringBuilder queryStringBuilder = new StringBuilder();
        boolean hasQuery = !queryParams.isEmpty();
        for (Map.Entry<String, Collection<String>> stringCollectionEntry : queryParams.entrySet()) {
            String key = stringCollectionEntry.getKey();
            Collection<String> queryParamsValueList = stringCollectionEntry.getValue();
            if (request.isQueryParamsParseAsMultiValue()) {
                for (String queryParamsValue : queryParamsValueList) {
                    uriBuilder.addParameter(key, queryParamsValue);
                    queryStringBuilder.append(key).append("=").append(queryParamsValue).append("&");
                }//from  w  ww. j a  v a  2  s. c  o  m
            } else {
                String value = joiner.join(queryParamsValueList);
                uriBuilder.addParameter(key, value);
                queryStringBuilder.append(key).append("=").append(value).append("&");
            }
        }
        uriBuilder.setFragment(requestUri.getFragment());
        uriBuilder.setHost(requestUri.getHost());
        uriBuilder.setPath(requestUri.getPath());
        uriBuilder.setPort(requestUri.getPort());
        uriBuilder.setScheme(requestUri.getScheme());
        uriBuilder.setUserInfo(requestUri.getUserInfo());
        try {

            if (!autoEncodeUri) {
                String urlPath = "";
                if (requestUri.getRawPath() != null && requestUri.getRawPath().startsWith("/")) {
                    urlPath = requestUri.getRawPath();
                } else {
                    urlPath = "/" + requestUri.getRawPath();
                }

                if (hasQuery) {
                    String query = queryStringBuilder.substring(0, queryStringBuilder.length() - 1);
                    requestUri = new URI(requestUri.getScheme() + "://" + requestUri.getHost() + ":"
                            + requestUri.getPort() + urlPath + "?" + query);
                } else {
                    requestUri = new URI(requestUri.getScheme() + "://" + requestUri.getHost() + ":"
                            + requestUri.getPort() + urlPath);
                }
            } else {
                requestUri = uriBuilder.build();
            }
        } catch (URISyntaxException e) {
            LOGGER.warn("could not update uri: {}", requestUri);
        }
    }
    return requestUri;
}

---