List of usage examples for org.jsoup.nodes Attributes get
public String get(String key)
From source file:com.astamuse.asta4d.render.RenderUtil.java
public final static void applyMessages(Element target) { Context context = Context.getCurrentThreadContext(); List<Element> msgElems = target.select(ExtNodeConstants.MSG_NODE_TAG_SELECTOR); for (final Element msgElem : msgElems) { Attributes attributes = msgElem.attributes(); String key = attributes.get(ExtNodeConstants.MSG_NODE_ATTR_KEY); // List<String> externalizeParamKeys = getExternalizeParamKeys(attributes); Object defaultMsg = new Object() { @Override/*from w ww.j a v a2 s. c o m*/ public String toString() { return ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX + msgElem.html(); } }; Locale locale = LocalizeUtil.getLocale(attributes.get(ExtNodeConstants.MSG_NODE_ATTR_LOCALE)); String currentTemplatePath = attributes.get(ExtNodeConstants.ATTR_TEMPLATE_PATH); if (StringUtils.isEmpty(currentTemplatePath)) { logger.warn("There is a msg tag which does not hold corresponding template file path:{}", msgElem.outerHtml()); } else { context.setData(TRACE_VAR_TEMPLATE_PATH, currentTemplatePath); } final Map<String, Object> paramMap = getMessageParams(attributes, locale, key); String text; switch (I18nMessageHelperTypeAssistant.configuredHelperType()) { case Mapped: text = I18nMessageHelperTypeAssistant.getConfiguredMappedHelper().getMessageWithDefault(locale, key, defaultMsg, paramMap); break; case Ordered: default: // convert map to array List<Object> numberedParamNameList = new ArrayList<>(); for (int index = 0; paramMap .containsKey(ExtNodeConstants.MSG_NODE_ATTR_PARAM_PREFIX + index); index++) { numberedParamNameList.add(paramMap.get(ExtNodeConstants.MSG_NODE_ATTR_PARAM_PREFIX + index)); } text = I18nMessageHelperTypeAssistant.getConfiguredOrderedHelper().getMessageWithDefault(locale, key, defaultMsg, numberedParamNameList.toArray()); } Node node; if (text.startsWith(ExtNodeConstants.MSG_NODE_ATTRVALUE_TEXT_PREFIX)) { node = ElementUtil.text(text.substring(ExtNodeConstants.MSG_NODE_ATTRVALUE_TEXT_PREFIX.length())); } else if (text.startsWith(ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX)) { node = ElementUtil .parseAsSingle(text.substring(ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX.length())); } else { node = ElementUtil.text(text); } msgElem.replaceWith(node); context.setData(TRACE_VAR_TEMPLATE_PATH, null); } }
From source file:com.romeikat.datamessie.core.base.service.download.ContentDownloader.java
public DownloadResult downloadContent(String url) { LOG.debug("Downloading content from {}", url); // In case of a new redirection for that source, use redirected URL URLConnection urlConnection = null; String originalUrl = null;// w w w .j av a 2 s. co m org.jsoup.nodes.Document jsoupDocument = null; Integer statusCode = null; final LocalDateTime downloaded = LocalDateTime.now(); try { urlConnection = getConnection(url); // Server-side redirection final String responseUrl = getResponseUrl(urlConnection); if (responseUrl != null) { final String redirectedUrl = getRedirectedUrl(url, responseUrl); if (isValidRedirection(url, redirectedUrl)) { originalUrl = url; url = redirectedUrl; closeUrlConnection(urlConnection); urlConnection = getConnection(url); LOG.debug("Redirection (server): {} -> {}", originalUrl, url); } } // Download content for further redirects final InputStream urlInputStream = asInputStream(urlConnection, true, false); final Charset charset = getCharset(urlConnection); jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url); final Elements metaTagsHtmlHeadLink; Elements metaTagsHtmlHeadMeta = null; // Meta redirection (<link rel="canonical" .../>) if (originalUrl == null) { metaTagsHtmlHeadLink = jsoupDocument.select("html head link"); for (final Element metaTag : metaTagsHtmlHeadLink) { final Attributes metaTagAttributes = metaTag.attributes(); if (metaTagAttributes.hasKey("rel") && metaTagAttributes.get("rel").equalsIgnoreCase("canonical") && metaTagAttributes.hasKey("href")) { final String redirectedUrl = metaTagAttributes.get("href").trim(); if (isValidRedirection(url, redirectedUrl)) { originalUrl = url; url = redirectedUrl; jsoupDocument = null; LOG.debug("Redirection (<link rel=\"canonical\" .../>): {} -> {}", originalUrl, url); break; } } } } // Meta redirection (<meta http-equiv="refresh" .../>) if (originalUrl == null) { metaTagsHtmlHeadMeta = jsoupDocument.select("html head meta"); for (final Element metaTag : metaTagsHtmlHeadMeta) { final Attributes metaTagAttributes = metaTag.attributes(); if (metaTagAttributes.hasKey("http-equiv") && metaTagAttributes.get("http-equiv").equalsIgnoreCase("refresh") && metaTagAttributes.hasKey("content")) { final String[] parts = metaTagAttributes.get("content").replace(" ", "").split("=", 2); if (parts.length > 1) { final String redirectedUrl = parts[1]; if (isValidRedirection(url, redirectedUrl)) { originalUrl = url; url = redirectedUrl; jsoupDocument = null; LOG.debug("Redirection (<meta http-equiv=\"refresh\" .../>): {} -> {}", originalUrl, url); break; } } } } } // Meta redirection (<meta property="og:url" .../>) if (originalUrl == null) { for (final Element metaTag : metaTagsHtmlHeadMeta) { final Attributes metaTagAttributes = metaTag.attributes(); if (metaTagAttributes.hasKey("property") && metaTagAttributes.get("property").equalsIgnoreCase("og:url") && metaTagAttributes.hasKey("content")) { final String redirectedUrl = metaTagAttributes.get("content").trim(); if (isValidRedirection(url, redirectedUrl)) { originalUrl = url; url = redirectedUrl; jsoupDocument = null; LOG.debug("Redirection (<meta property=\"og:url\" .../>): {} -> {}", originalUrl, url); break; } } } } } catch (final Exception e) { if (e instanceof HttpStatusException) { statusCode = ((HttpStatusException) e).getStatusCode(); } LOG.warn("Could not determine redirected URL for " + url, e); } finally { closeUrlConnection(urlConnection); } // Download content (if not yet done) String content = null; try { if (jsoupDocument == null) { LOG.debug("Downloading content from {}", url); urlConnection = getConnection(url); final InputStream urlInputStream = asInputStream(urlConnection, true, false); final Charset charset = getCharset(urlConnection); jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url); } } catch (final Exception e) { if (e instanceof HttpStatusException) { statusCode = ((HttpStatusException) e).getStatusCode(); } // If the redirected URL does not exist, use the original URL instead if (originalUrl == null) { LOG.warn("Could not download content from " + url, e); } // If the redirected URL does not exist and a original URL is available, use the // original URL instead else { try { LOG.debug( "Could not download content from redirected URL {}, downloading content from original URL {} instead", url, originalUrl); urlConnection = getConnection(originalUrl); final InputStream urlInputStream = asInputStream(urlConnection, true, false); final Charset charset = getCharset(urlConnection); jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url); url = originalUrl; originalUrl = null; statusCode = null; } catch (final Exception e2) { LOG.warn("Could not download content from original URL " + url, e); } } } finally { closeUrlConnection(urlConnection); } if (jsoupDocument != null) { content = jsoupDocument.html(); } // Strip non-valid characters as specified by the XML 1.0 standard final String validContent = xmlUtil.stripNonValidXMLCharacters(content); // Unescape HTML characters final String unescapedContent = StringEscapeUtils.unescapeHtml4(validContent); // Done final DownloadResult downloadResult = new DownloadResult(originalUrl, url, unescapedContent, downloaded, statusCode); return downloadResult; }
From source file:net.vexelon.mobileops.GLBClient.java
private List<NameValuePair> findLoginParams() throws HttpClientException { List<NameValuePair> result = new ArrayList<NameValuePair>(); BufferedReader reader = null; long bytesCount = 0; try {/*ww w.j a va2 s .c om*/ // Get invoice check page StringBuilder fullUrl = new StringBuilder(100); fullUrl.append(GLBRequestType.LOGIN.getPath()).append("?").append(GLBRequestType.LOGIN.getParams()); HttpGet httpGet = new HttpGet(fullUrl.toString()); HttpResponse resp = httpClient.execute(httpGet, httpContext); StatusLine status = resp.getStatusLine(); if (status.getStatusCode() == HttpStatus.SC_OK) { Document doc = Jsoup.parse(resp.getEntity().getContent(), RESPONSE_ENCODING, ""); Elements inputs = doc.select("input"); for (Element el : inputs) { // if (Defs.LOG_ENABLED) { // Log.v(Defs.LOG_TAG, "ELEMENT: " + el.tagName()); // } Attributes attrs = el.attributes(); // for (Attribute attr : attrs) { // if (Defs.LOG_ENABLED) { // Log.v(Defs.LOG_TAG, " " + attr.getKey() + "=" + attr.getValue()); // } // } String elName = attrs.get("name"); if (elName.equalsIgnoreCase("lt") || elName.equalsIgnoreCase("execution")) { result.add(new BasicNameValuePair(elName, attrs.get("value"))); } } } else { throw new HttpClientException(status.getReasonPhrase(), status.getStatusCode()); } // close current stream reader // if (reader != null) try { reader.close(); } catch (IOException e) {}; } catch (ClientProtocolException e) { throw new HttpClientException("Client protocol error!" + e.getMessage(), e); } catch (IOException e) { throw new HttpClientException("Client error!" + e.getMessage(), e); } finally { if (reader != null) try { reader.close(); } catch (IOException e) { } ; addDownloadedBytesCount(bytesCount); } return result; }