Example usage for org.apache.http.client.utils URLEncodedUtils parse

List of usage examples for org.apache.http.client.utils URLEncodedUtils parse

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.client.utils.*
  5. URLEncodedUtils
  6. parse

Introduction

In this page you can find the example usage for org.apache.http.client.utils URLEncodedUtils parse.

Prototype

public static List<NameValuePair> parse(final String s, final Charset charset,
        final char... parameterSeparator)

Source Link

Document

Returns a list of NameValuePair NameValuePairs as parsed from the given string using the given character encoding.

Usage

From source file:org.github.oauth2.AccessTokenClient.java

/**
 * Get token from resposne entity/*  w w w .j  a v a2s.c om*/
 * 
 * @param entity
 * @return token or null if not present in given entity
 * @throws IOException
 */
protected String getToken(HttpEntity entity) throws IOException {
    String content = EntityUtils.toString(entity, HTTP.ASCII);
    if (content == null || content.length() == 0)
        return null;
    List<NameValuePair> responseData = new ArrayList<NameValuePair>();
    URLEncodedUtils.parse(responseData, new Scanner(content), null);
    for (NameValuePair param : responseData)
        if (IOAuth2Constants.PARAM_ACCESS_TOKEN.equals(param.getName())) {
            String token = param.getValue();
            if (token != null && token.length() > 0)
                return token;
        }
    return null;
}

From source file:com.nebkat.plugin.youtube.YoutubeRetriever.java

private static Map<String, String> getNameValuePairMap(String queryString) {
    List<NameValuePair> infoMap = new ArrayList<>();
    URLEncodedUtils.parse(infoMap, new Scanner(queryString), "UTF-8");
    Map<String, String> map = new HashMap<>(infoMap.size());
    infoMap.forEach((pair) -> map.put(pair.getName(), pair.getValue()));
    return map;/*ww  w  .  jav  a 2 s .  c  o  m*/
}

From source file:com.twosigma.beaker.sql.ConnectionStringHolder.java

/**
 * MSSQL driver do not return password, so we need to parse it manually
 * @param property//from  w w  w.  j a v  a2  s  . com
 * @param connectionString
 * @return
 */
protected static String getProperty(String property, String connectionString) {
    String ret = null;
    if (property != null && !property.isEmpty() && connectionString != null && !connectionString.isEmpty()) {
        for (NameValuePair param : URLEncodedUtils.parse(connectionString, StandardCharsets.UTF_8,
                SEPARATORS)) {
            if (property.equals(param.getName())) {
                ret = param.getValue();
                break;
            }
        }
    }
    return ret;
}

From source file:com.digitalpebble.storm.crawler.filtering.basic.BasicURLNormalizer.java

/**
 * Basic filter to remove query parameters from urls so parameters that
 * don't change the content of the page can be removed. An example would be
 * a google analytics query parameter like "utm_campaign" which might have
 * several different values for a url that points to the same content.
 *///from ww w  .  j  av  a 2 s . co  m
private String filterQueryElements(String urlToFilter) {
    try {
        // Handle illegal characters by making a url first
        // this will clean illegal characters like |
        URL url = new URL(urlToFilter);

        if (StringUtils.isEmpty(url.getQuery())) {
            return urlToFilter;
        }

        List<NameValuePair> pairs = new ArrayList<NameValuePair>();
        URLEncodedUtils.parse(pairs, new Scanner(url.getQuery()), "UTF-8");
        Iterator<NameValuePair> pairsIterator = pairs.iterator();
        while (pairsIterator.hasNext()) {
            NameValuePair param = pairsIterator.next();
            if (queryElementsToRemove.contains(param.getName())) {
                pairsIterator.remove();
            }
        }

        StringBuilder newFile = new StringBuilder();
        if (url.getPath() != null) {
            newFile.append(url.getPath());
        }
        if (!pairs.isEmpty()) {
            Collections.sort(pairs, comp);
            String newQueryString = URLEncodedUtils.format(pairs, StandardCharsets.UTF_8);
            newFile.append('?').append(newQueryString);
        }
        if (url.getRef() != null) {
            newFile.append('#').append(url.getRef());
        }

        return new URL(url.getProtocol(), url.getHost(), url.getPort(), newFile.toString()).toString();
    } catch (MalformedURLException e) {
        LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
        return null;
    }
}

From source file:com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer.java

/**
 * Basic filter to remove query parameters from urls so parameters that
 * don't change the content of the page can be removed. An example would be
 * a google analytics query parameter like "utm_campaign" which might have
 * several different values for a url that points to the same content.
 *///from   w w w. j a v a 2  s .  co m
private String filterQueryElements(String urlToFilter) {
    try {
        // Handle illegal characters by making a url first
        // this will clean illegal characters like |
        URL url = new URL(urlToFilter);

        if (StringUtils.isEmpty(url.getQuery())) {
            return urlToFilter;
        }

        List<NameValuePair> pairs = new ArrayList<>();
        URLEncodedUtils.parse(pairs, new Scanner(url.getQuery()), "UTF-8");
        Iterator<NameValuePair> pairsIterator = pairs.iterator();
        while (pairsIterator.hasNext()) {
            NameValuePair param = pairsIterator.next();
            if (queryElementsToRemove.contains(param.getName())) {
                pairsIterator.remove();
            }
        }

        StringBuilder newFile = new StringBuilder();
        if (url.getPath() != null) {
            newFile.append(url.getPath());
        }
        if (!pairs.isEmpty()) {
            Collections.sort(pairs, comp);
            String newQueryString = URLEncodedUtils.format(pairs, StandardCharsets.UTF_8);
            newFile.append('?').append(newQueryString);
        }
        if (url.getRef() != null) {
            newFile.append('#').append(url.getRef());
        }

        return new URL(url.getProtocol(), url.getHost(), url.getPort(), newFile.toString()).toString();
    } catch (MalformedURLException e) {
        LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
        return null;
    }
}

From source file:com.tri_voltage.md.io.YoutubeVideoParser.java

private static void play(String videoId, int format, String encoding, String userAgent, File outputdir,
        String extension) throws Throwable {
    log.fine("Retrieving " + videoId);
    List<NameValuePair> qparams = new ArrayList<NameValuePair>();
    qparams.add(new BasicNameValuePair("video_id", videoId));
    qparams.add(new BasicNameValuePair("fmt", "" + format));
    URI uri = getUri("get_video_info", qparams);

    CookieStore cookieStore = new BasicCookieStore();
    HttpContext localContext = new BasicHttpContext();
    localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);

    HttpClient httpclient = HttpClientBuilder.create().build();
    HttpGet httpget = new HttpGet(uri);
    if (userAgent != null && userAgent.length() > 0) {
        httpget.setHeader("User-Agent", userAgent);
    }/*from w  w  w .  j a  v a2 s .  com*/

    log.finer("Executing " + uri);
    HttpResponse response = httpclient.execute(httpget, localContext);
    HttpEntity entity = response.getEntity();
    if (entity != null && response.getStatusLine().getStatusCode() == 200) {
        InputStream instream = entity.getContent();
        String videoInfo = getStringFromInputStream(encoding, instream);
        if (videoInfo != null && videoInfo.length() > 0) {
            List<NameValuePair> infoMap = new ArrayList<NameValuePair>();
            URLEncodedUtils.parse(infoMap, new Scanner(videoInfo), encoding);
            String downloadUrl = null;
            String filename = videoId;

            for (NameValuePair pair : infoMap) {
                String key = pair.getName();
                String val = pair.getValue();
                log.finest(key + "=" + val);
                if (key.equals("title")) {
                    filename = val;
                } else if (key.equals("fmt_url_map")) {
                    String[] formats = commaPattern.split(val);
                    boolean found = false;
                    for (String fmt : formats) {
                        String[] fmtPieces = pipePattern.split(fmt);
                        if (fmtPieces.length == 2) {
                            int pieceFormat = Integer.parseInt(fmtPieces[0]);
                            log.fine("Available format=" + pieceFormat);
                            if (pieceFormat == format) {
                                // found what we want
                                downloadUrl = fmtPieces[1];
                                found = true;
                                break;
                            }
                        }
                    }
                    if (!found) {
                        log.warning(
                                "Could not find video matching specified format, however some formats of the video do exist (use -verbose).");
                    }
                }
            }

            filename = cleanFilename(filename);
            if (filename.length() == 0) {
                filename = videoId;
            } else {
                filename += "_" + videoId;
            }
            filename += "." + extension;
            File outputfile = new File(outputdir, filename);

            if (downloadUrl != null) {
                downloadWithHttpClient(userAgent, downloadUrl, outputfile);
            } else {
                log.severe("Could not find video");
            }
        } else {
            log.severe("Did not receive content from youtube");
        }
    } else {
        log.severe("Could not contact youtube: " + response.getStatusLine());
    }
}

From source file:com.subgraph.vega.internal.model.web.WebPath.java

private static List<NameValuePair> parseParameters(String query) {
    if (query == null || query.isEmpty())
        return Collections.emptyList();
    final List<NameValuePair> parameterList = new ArrayList<NameValuePair>();
    try {/*from  w w w. j  av a 2  s.  c o m*/
        URLEncodedUtils.parse(parameterList, new Scanner(query), "UTF-8");
    } catch (RuntimeException e) {
        parameterList.clear();
        parameterList.add(new BasicNameValuePair(query, null));
    }
    return parameterList;
}

From source file:gtu.youtube.JavaYoutubeDownloader.java

private void play(String videoId, int format, String encoding, String userAgent, File outputdir,
        String extension) throws Throwable {
    // ?Youtube?/*from w  ww . j a va  2s. co m*/
    JavaYoutubeVideoUrlHandler urlHandler = new JavaYoutubeVideoUrlHandler(videoId, String.valueOf(format),
            userAgent);

    // ?
    log.fine("Retrieving " + videoId);
    List<NameValuePair> qparams = new ArrayList<NameValuePair>();
    qparams.add(new BasicNameValuePair("video_id", videoId));
    qparams.add(new BasicNameValuePair("fmt", "" + format));
    URI uri = getUri("get_video_info", qparams);

    CookieStore cookieStore = new BasicCookieStore();
    if (true) {
        InputStream is = this.getClass().getResource("JavaYoutubeDownloader_cookies.txt").openStream();
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        IOUtils.copy(is, baos);
        String cookieContent = baos.toString("UTF-8");
        cookieStore = this.getCookieString(cookieContent);
    }

    HttpContext localContext = new BasicHttpContext();
    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

    HttpClient httpclient = new DefaultHttpClient();
    HttpGet httpget = new HttpGet(uri);
    if (userAgent != null && userAgent.length() > 0) {
        httpget.setHeader("User-Agent", userAgent);
    }

    log.finer("Executing " + uri);
    HttpResponse response = httpclient.execute(httpget, localContext);
    HttpEntity entity = response.getEntity();
    if (entity != null && response.getStatusLine().getStatusCode() == 200) {
        InputStream instream = entity.getContent();
        String videoInfo = getStringFromInputStream(encoding, instream);
        System.out.println("videoInfo = " + videoInfo);
        if (videoInfo != null && videoInfo.length() > 0) {
            List<NameValuePair> infoMap = new ArrayList<NameValuePair>();
            URLEncodedUtils.parse(infoMap, new Scanner(videoInfo), encoding);
            System.out.println("infoMap = " + infoMap);

            String downloadUrl = null;
            String filename = videoId;

            for (NameValuePair pair : infoMap) {
                String key = pair.getName();
                String val = pair.getValue();
                log.finest(key + "=" + val);
                // System.out.println(key + "\t" + val);
                if (key.equals("title")) {
                    filename = val;
                } else if (key.equals("fmt_url_map")) { // 
                    String[] formats = commaPattern.split(val);
                    boolean found = false;
                    for (String fmt : formats) {
                        String[] fmtPieces = pipePattern.split(fmt);
                        if (fmtPieces.length == 2) {
                            int pieceFormat = Integer.parseInt(fmtPieces[0]);
                            log.fine("Available format=" + pieceFormat);
                            if (pieceFormat == format) {
                                // found what we want
                                downloadUrl = fmtPieces[1];
                                System.out.println(">>> downloadUrl = " + downloadUrl);
                                found = true;
                                break;
                            }
                        }
                    }
                    if (!found) {
                        log.warning(
                                "Could not find video matching specified format, however some formats of the video do exist (use -verbose).");
                    }
                }
            }

            filename = cleanFilename(filename);
            if (filename.length() == 0) {
                filename = videoId;
            } else {
                filename += "_" + videoId;
            }
            filename += "." + extension;
            File outputfile = new File(outputdir, filename);

            // ?1
            if (downloadUrl == null) {
                downloadUrl = customDownloadUrl_by_gtu001;
            }

            // ?2
            if (downloadUrl == null) {
                JavaYoutubeVideoUrlHandler gtu001 = new JavaYoutubeVideoUrlHandler(videoId, "", userAgent);
                downloadUrl = gtu001.getUrl(format);
            }

            if (downloadUrl != null) {
                downloadWithHttpClient(userAgent, downloadUrl, outputfile);
            } else {
                log.severe("Could not find video");
            }
        } else {
            log.severe("Did not receive content from youtube");
        }
    } else {
        log.severe("Could not contact youtube: " + response.getStatusLine());
    }
}

From source file:gtu.youtube.JavaYoutubeVideoUrlHandler.java

public List<NameValuePair> getVideoInfo(String videoId, String format, String userAgent) {
    try {/*from w  w  w . j  a v a2  s  .co  m*/
        List<NameValuePair> infoMap = new ArrayList<NameValuePair>();
        List<NameValuePair> qparams = new ArrayList<NameValuePair>();
        qparams.add(new BasicNameValuePair("video_id", videoId));
        qparams.add(new BasicNameValuePair("fmt", "" + format));
        URI uri = getUri("get_video_info", qparams);

        if (StringUtils.isBlank(userAgent)) {
            userAgent = DEFAULT_USER_AGENT;
        }

        CookieStore cookieStore = new BasicCookieStore();
        HttpContext localContext = new BasicHttpContext();
        localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

        HttpClient httpclient = new DefaultHttpClient();
        HttpGet httpget = new HttpGet(uri);
        if (userAgent != null && userAgent.length() > 0) {
            httpget.setHeader("User-Agent", userAgent);
        }

        HttpResponse response = httpclient.execute(httpget, localContext);
        HttpEntity entity = response.getEntity();
        if (entity != null && response.getStatusLine().getStatusCode() == 200) {
            InputStream instream = entity.getContent();
            String videoInfo = getStringFromInputStream(DEFAULT_ENCODING, instream);
            URLEncodedUtils.parse(infoMap, new Scanner(videoInfo), DEFAULT_ENCODING);
        }
        return infoMap;
    } catch (Exception ex) {
        throw new RuntimeException("play Err : " + ex.getMessage(), ex);
    }
}