List of usage examples for org.jsoup.nodes Document html
public String html()
From source file:com.mohawk.webcrawler.lang.verb.GetUrl_Verb.java
@Override public Object run(ScriptContext pageContext, Object... params) throws Exception { Object p1 = LangCore.resolveParameter(pageContext, params[0]); if (!(p1 instanceof String)) throw new LanguageException("String literal required as parameter>> " + p1); String url = (String) p1; Document doc = null; Config config = pageContext.getConfig(); if (config.getCacheDirectory() != null) { // pull the HTML from cache String prefix = null; //pageContext.getConfig().getProviderId() + "_"; Collection<File> files = FileUtils.listFiles( new File("C:\\Users\\cnguyen\\Projects\\ProjectMohawk\\Scripts\\cache"), null, false); for (File file : files) { if (file.getName().startsWith(prefix)) { doc = Jsoup.parse(file, "UTF-8"); break; }/*w w w. j a v a2 s .c om*/ } } else { int MINS_2 = 2 * 60 * 1000; doc = (Document) Jsoup.parse(new URL(url), MINS_2); } // clear out any other contexts pageContext.setTableContext(null); pageContext.setDocument(doc); pageContext.setDocumentHtml(doc.html()); pageContext.setCursorPosition(0); return null; }
From source file:com.mohawk.webcrawler.lang.verb.GetPdf_Verb.java
@Override public Object run(ScriptContext pageContext, Object... params) throws Exception { String dataDirectory = System.getenv("OPENSHIFT_DATA_DIR"); String cacheDirectory = null; if (dataDirectory != null) cacheDirectory = dataDirectory + "webcrawler/cache"; else//from w w w. ja va 2s. c om cacheDirectory = "C:\\Users\\cnguyen\\Projects\\ProjectMohawk\\Scripts\\cache"; Object param = LangCore.resolveParameter(pageContext, params[0]); Document doc = null; Config config = pageContext.getConfig(); if (config.getCacheDirectory() != null) { // pull the HTML from cache String prefix = null; //config.getProviderId() + "_"; Collection<File> files = FileUtils.listFiles(new File(cacheDirectory), null, false); for (File file : files) { String filename = file.getName(); if (file.getName().startsWith(prefix) && filename.endsWith(".html")) { doc = (Document) Jsoup.parse(file, "UTF-8"); break; } } if (doc == null) throw new Exception("Unable to find cached file for script>> " + config.getScriptFilename()); } else { // get it from the web System.out.println("URL>> " + param); } // clear out any other contexts pageContext.setTableContext(null); pageContext.setDocument(doc); pageContext.setDocumentHtml(doc.html()); pageContext.setCursorPosition(0); return null; }
From source file:com.isomorphic.maven.packaging.Downloads.java
/** * Interrogates the remote server for a list of hyperlinks matching the given distribution's {@link Distribution#getRemoteIndexFilter() filter}. * // w w w . j a v a2s.c o m * @param dist the build in which some files should exist * @return a String array of html href attributes * @throws MojoExecutionException */ private String[] list(Distribution dist) throws MojoExecutionException { HttpGet request = new HttpGet(dist.getRemoteIndex()); HttpResponse response; try { LOGGER.debug("Requesting list of files from {}{}", DOMAIN, dist.getRemoteIndex()); response = httpClient.execute(host, request); } catch (Exception e) { throw new MojoExecutionException("Error issuing GET request for bundle at '" + request + "'", e); } Document doc; try { String html = EntityUtils.toString(response.getEntity()); doc = Jsoup.parse(html); doc.outputSettings().prettyPrint(true); } catch (Exception e) { throw new MojoExecutionException("Error processing response from '" + request + "'", e); } List<String> result = new ArrayList<String>(); Elements links = doc.select(dist.getRemoteIndexFilter()); for (Element element : links) { String href = element.attr("href"); result.add(href); } if (result.isEmpty()) { String msg = String.format("No downloads found at '%s%s'. Response from server: \n\n%s\n", DOMAIN, dist.getRemoteIndex(), doc.html()); LOGGER.warn(msg); } return result.toArray(new String[0]); }
From source file:codeu.chat.client.commandline.Chat.java
private boolean parseScript(String link, String phrase, boolean springfield) { String[] script;//ww w . j ava 2s.c o m try { Document doc = Jsoup.connect(link).get(); /* If the script was retrieved from the Springfield website, the lines must be split up using the <br> tag instead of new line characters */ if (springfield) { String temp = Jsoup.parse(doc.html().replaceAll("(?i)<br[^>]*>", "br2n")).text(); script = mergeScriptSentences(temp.split("br2n")); } else { script = mergeScriptSentences(doc.body().text().split("\n")); } /* Search for a line containing the phrase. Once one is found, determine the best response and return accordingly. In some cases, this will mean continuing to search for a later match */ for (int lineNum = 0; lineNum < script.length; lineNum++) { script[lineNum] = script[lineNum].trim().toLowerCase(); for (String sentence : script[lineNum].split("(?<=[!\\?\\.])")) { if (sentence.contains(phrase) || StringUtils.getLevenshteinDistance(sentence, phrase) <= phrase.length() / 3.0) { if (findNextScriptResponse(lineNum, phrase, script)) { return true; } } } } } catch (IOException e) { e.printStackTrace(); } return false; // Return false if no line containing the phrase was found }
From source file:net.slkdev.swagger.confluence.service.impl.XHtmlToConfluenceServiceImpl.java
private static String reformatXHtml(final String inputXhtml, final Map<String, ConfluenceLink> confluenceLinkMap) { final Document document = Jsoup.parse(inputXhtml, "utf-8", Parser.xmlParser()); document.outputSettings().prettyPrint(false); document.outputSettings().escapeMode(xhtml); document.outputSettings().charset("UTF-8"); final Elements linkElements = document.select("a"); for (final Element linkElement : linkElements) { final String originalHref = linkElement.attr("href"); final ConfluenceLink confluenceLink = confluenceLinkMap.get(originalHref); if (confluenceLink == null) { LOG.debug("NO LINK MAPPING FOUND TO COVERT LINK: {}", originalHref); continue; }/* w w w . ja v a2 s .co m*/ final String confluenceLinkMarkup = confluenceLink.getConfluenceLinkMarkup(); LOG.debug("LINK CONVERSION: {} -> {}", originalHref, confluenceLinkMarkup); linkElement.before(confluenceLinkMarkup); linkElement.html(""); linkElement.unwrap(); } reformatXHtmlHeadings(document, "h2"); reformatXHtmlHeadings(document, "h3"); reformatXHtmlHeadings(document, "#toctitle"); final SwaggerConfluenceConfig swaggerConfluenceConfig = SWAGGER_CONFLUENCE_CONFIG.get(); if (swaggerConfluenceConfig.getPaginationMode() == PaginationMode.SINGLE_PAGE) { if (swaggerConfluenceConfig.isIncludeTableOfContentsOnSinglePage()) { reformatXHtmlBreakAfterElements(document, "#toc"); } reformatXHtmlBreakAfterElements(document, ".sect1"); } reformatXHtmlSpacing(document.select(".sect2")); reformatXHtmlSpacing(document.select(".sect3")); return document.html(); }
From source file:com.blackducksoftware.tools.nrt.generator.NRTReportGenerator.java
/** * Copies the HTML template into the finalHtmlOutput then injects the * generates JSON data into the specific div location and writes it out. * /*from w w w . ja v a 2 s .c om*/ * @param expectedFile */ public void generateHTMLFromTemplate(File finalHtmlOutput) { log.info("Writing to report: " + finalHtmlOutput); String jsonComponentList = generateJSONFromObject(componentMap); String jsonPropertyList = generateJSONFromObject(nrtConfig.getOptionsForExport()); // Construct a variable out of it jsonComponentList = "var compList=[" + jsonComponentList + "]"; jsonPropertyList = "var propList=[" + jsonPropertyList + "]"; PrintWriter writer = null; try { // Read the template Document doc = Jsoup.parse(finalHtmlOutput, "UTF-8"); // Inject the JSON Elements jsonElementDivBlock = doc.getElementsByClass(NRTConstants.HTML_JSON_DATA_BLOCK); // This will be empty, but it should exist Element jsonDivElement = jsonElementDivBlock.get(0); if (jsonDivElement != null) { // Remove any script tags from it, in case the user populated // the template incorrectly with data if (jsonDivElement.children().size() > 0) { Elements children = jsonDivElement.children(); for (int i = 0; i < children.size(); i++) { Element el = children.get(i); el.remove(); } } addNewScriptElementWithJson(jsonDivElement, jsonComponentList); addNewScriptElementWithJson(jsonDivElement, jsonPropertyList); } else { log.error("Unable to find a valid critical DIV inside HTML template: " + NRTConstants.HTML_JSON_DATA_BLOCK); } writer = new PrintWriter(finalHtmlOutput, "UTF-8"); // Write out the file writer.write(doc.html()); writer.flush(); writer.close(); } catch (Exception e) { log.error("Unable to write out final report file!", e); } finally { writer.close(); } }
From source file:de.geeksfactory.opacclient.apis.Zones.java
private SearchRequestResult parse_search(String html, int page) throws OpacErrorException { Document doc = Jsoup.parse(html); doc.setBaseUri(opac_url + "/APS_PRESENT_BIB"); if (doc.select("#ErrorAdviceRow").size() > 0) { throw new OpacErrorException(doc.select("#ErrorAdviceRow").text().trim()); }/*from w ww .j ava 2 s .c o m*/ int results_total = -1; String searchHitsQuery = version18 ? "td:containsOwn(Total)" : ".searchHits"; if (doc.select(searchHitsQuery).size() > 0) { results_total = Integer.parseInt( doc.select(searchHitsQuery).first().text().trim().replaceAll(".*\\(([0-9]+)\\).*", "$1")); } else if (doc.select("span:matches(\\[\\d+/\\d+\\])").size() > 0) { // Zones 1.8 - searchGetPage String text = doc.select("span:matches(\\[\\d+/\\d+\\])").text(); Pattern pattern = Pattern.compile("\\[\\d+/(\\d+)\\]"); Matcher matcher = pattern.matcher(text); if (matcher.find()) { results_total = Integer.parseInt(matcher.group(1)); } } if (doc.select(".pageNavLink").size() > 0) { // Zones 2.2 searchobj = doc.select(".pageNavLink").first().attr("href").split("\\?")[0]; } else if (doc.select("div[targetObject]").size() > 0) { // Zones 1.8 - search searchobj = doc.select("div[targetObject]").attr("targetObject").split("\\?")[0]; } else { // Zones 1.8 - searchGetPage // The page contains a data structure that at first glance seems to be JSON, but uses // "=" instead of ":". So we parse it using regex... Pattern pattern = Pattern.compile("targetObject = \"([^\\?]+)[^\"]+\""); Matcher matcher = pattern.matcher(doc.html()); if (matcher.find()) { searchobj = matcher.group(1); } } Elements table = doc.select("#BrowseList > tbody > tr," // Zones 2.2 + " .inRoundBox1" // Zones 1.8 ); List<SearchResult> results = new ArrayList<>(); for (int i = 0; i < table.size(); i++) { Element tr = table.get(i); SearchResult sr = new SearchResult(); String typetext; if (version18) { String[] parts = tr.select("img[src^=IMG/MAT]").attr("src").split("/"); typetext = parts[parts.length - 1].replace(".gif", ""); } else { typetext = tr.select(".SummaryMaterialTypeField").text().replace("\n", " ").trim(); } if (data.has("mediatypes")) { try { sr.setType(MediaType.valueOf(data.getJSONObject("mediatypes").getString(typetext))); } catch (JSONException | IllegalArgumentException e) { sr.setType(defaulttypes.get(typetext)); } } else { sr.setType(defaulttypes.get(typetext)); } String imgUrl = null; if (version18) { if (tr.select("a[title=Titelbild]").size() > 0) { imgUrl = tr.select("a[title=Titelbild]").attr("href"); } else if (tr.select("img[width=50]").size() > 0) { // TODO: better way to select these cover images? (found in Hannover) imgUrl = tr.select("img[width=50]").attr("src"); } } else { if (tr.select(".SummaryImageCell img[id^=Bookcover]").size() > 0) { imgUrl = tr.select(".SummaryImageCell img[id^=Bookcover]").first().attr("src"); } } sr.setCover(imgUrl); if (version18) { if (tr.select("img[src$=oci_1.gif]").size() > 0) { // probably can only appear when searching the catalog on a terminal in // the library. sr.setStatus(SearchResult.Status.GREEN); } else if (tr.select("img[src$=blob_amber.gif]").size() > 0) { sr.setStatus(SearchResult.Status.YELLOW); } } String desc = ""; String childrenQuery = version18 ? "table[cellpadding=1] tr" : ".SummaryDataCell tr, .SummaryDataCellStripe tr"; Elements children = tr.select(childrenQuery); int childrennum = children.size(); boolean haslink = false; for (int ch = 0; ch < childrennum; ch++) { Element node = children.get(ch); if (getName(node).equals("Titel")) { desc += "<b>" + getValue(node).trim() + "</b><br />"; } else if (getName(node).equals("Verfasser") || getName(node).equals("Jahr")) { desc += getValue(node).trim() + "<br />"; } String linkSelector = version18 ? "a[href*=ShowStock], a[href*=APS_CAT_IDENTIFY]" : ".SummaryFieldData a.SummaryFieldLink"; if (node.select(linkSelector).size() > 0 && !haslink) { String href = node.select(linkSelector).attr("abs:href"); Map<String, String> hrefq = getQueryParamsFirst(href); if (hrefq.containsKey("no")) { sr.setId(hrefq.get("no")); } else if (hrefq.containsKey("Key")) { sr.setId(hrefq.get("Key")); } haslink = true; } } if (desc.endsWith("<br />")) { desc = desc.substring(0, desc.length() - 6); } sr.setInnerhtml(desc); sr.setNr(i); results.add(sr); } return new SearchRequestResult(results, results_total, page); }
From source file:com.ibuildapp.romanblack.WebPlugin.WebPlugin.java
/** * Prepare and load data to WebView.//from w w w . j av a2 s. c o m */ private void showHtml() { try { if (isOnline) { if (currentUrl.length() > 0 && !currentUrl.equals("about:blank")) { url = currentUrl; } if (url.length() > 0) html = "<html><body><a href=\"" + url + "\" id=\"link\" /></body></html>"; Document doc = Jsoup.parse(html); Element iframe = doc.select("iframe").first(); boolean isGoogleCalendar = false; boolean isGoogleForms = false; String iframeSrc = ""; try { if (iframe != null) { iframeSrc = iframe.attr("src"); } } catch (Exception e) { } if (iframeSrc.length() > 0) { isGoogleCalendar = iframeSrc.contains("www.google.com/calendar") || iframeSrc.contains("calendar.google.com/calendar"); isGoogleForms = iframeSrc.contains("google.com/forms"); } if (isGoogleCalendar) { webView.loadUrl(iframeSrc); } else if (isGoogleForms) { webView.getSettings().setBuiltInZoomControls(false); DisplayMetrics metrix = getResources().getDisplayMetrics(); int width = metrix.widthPixels; int height = metrix.heightPixels; float density = metrix.density; iframe.attr("width", (int) (width / density) + ""); iframe.attr("height", (int) (height / density - (75 /*+ (hasAdView() ? 50 : 0)*/)) + ""); iframe.attr("style", "margin: 0; padding: 0"); Element body = doc.select("body").first(); body.attr("style", "margin: 0; padding: 0"); html = doc.outerHtml(); webView.loadDataWithBaseURL("http://", html, "text/html", "utf-8", ""); } else { Elements forms = doc.select("form"); Iterator<Element> iterator = forms.iterator(); for (; iterator.hasNext();) { Element form = iterator.next(); String action = form.attr("action"); if (action.contains("paypal.com")) { form.append("<input type=\"hidden\" name=\"bn\" value=\"ibuildapp_SP\">"); } html = doc.html(); } hideProgress = true; if (Build.VERSION.SDK_INT >= 20 && html.contains("ibuildapp") && html.contains("powr")) { int height = getResources().getDisplayMetrics().heightPixels; html = "<iframe width=\"" + 420 + "\" height=\"" + height + "\" frameBorder=\"0\" src=" + url + "></iframe>"; webView.loadData(html, "text/html", "utf-8"); } else webView.loadDataWithBaseURL("http://", html, "text/html", "utf-8", ""); } } else { if (html.length() > 0) { webView.loadDataWithBaseURL("http://", html, "text/html", "utf-8", ""); } } handler.sendEmptyMessageDelayed(HIDE_PROGRESS, 10000); } catch (Exception ex) { // Error Logging } }
From source file:com.weavers.duqhan.business.impl.ProductServiceImpl.java
@Override public void loadTempProducts(List<StatusBean> statusBeans) { boolean isSuccess = true; String startDate = new Date().toString(); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==I==)DATE: " + startDate + "Store product details in temp product table start....."); try {/* w ww . j a va 2 s. com*/ String status = ""; for (StatusBean statusBean : statusBeans) { status = "Link duplicate"; Temtproductlinklist temtproductlinklist = temtproductlinklistDao.loadById(statusBean.getId()); if (temtproductlinklist != null && temtproductlinklist.getStatus() == 0) { Product testProduct = productDao.getProductByExternelLink(temtproductlinklist.getLink()); if (testProduct == null) { String value = ""; Elements detailMain; Elements detailSub; Elements specifics; double votes = 0.0; double stars = 0.0; double feedback = 0.0; String url = temtproductlinklist.getLink(); try { testProduct = new Product(); Product savedTestProduct; //=================== Random sleep START ===================// // TimeUnit.SECONDS.sleep(30 + (int) (Math.random() * 100)); Random randomObj = new Random(); TimeUnit.SECONDS.sleep(randomObj.ints(30, 60).findFirst().getAsInt()); //=================== Random sleep END =====================// Document doc = Jsoup.connect(url).get(); detailMain = doc.select("#j-detail-page"); if (!detailMain.isEmpty()) { //=================== Criteria Block START==================// detailMain = doc.select(".rantings-num"); if (!detailMain.isEmpty()) { votes = Double.valueOf(detailMain.text().split(" votes")[0].split("\\(")[1]); } detailMain = doc.select(".percent-num"); if (!detailMain.isEmpty()) { stars = Double.valueOf(detailMain.text()); } detailMain = doc.select("ul.ui-tab-nav li[data-trigger='feedback'] a"); if (!detailMain.isEmpty()) { feedback = Double.valueOf(detailMain.text().split("\\(")[1].split("\\)")[0]); } //=================== Criteria Block END==================// if (votes > 10.0 && stars > 4.0 && feedback > 4.0) { detailMain = doc.select(".detail-wrap .product-name"); testProduct.setName(detailMain .text());/*.substring(0, Math.min(detailMain.text().length(), 50))*/ detailMain = doc.select(".detail-wrap .product-name"); testProduct.setDescription(detailMain.text()); testProduct.setExternalLink(url); testProduct.setVendorId(1l);//?????????????????????? //=================== Packaging block START==================// Double weight = 1.0; Double width = 1.0; Double height = 1.0; Double length = 1.0; detailMain = doc.select( "div#j-product-desc div.pnl-packaging-main ul li.packaging-item"); for (Element element : detailMain) { String packagingTitle = element.select("span.packaging-title").text(); String packagingDesc = element.select("span.packaging-des").text(); if (packagingTitle.trim().equals("Package Weight:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { weight = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); } catch (Exception e) { weight = 1.0; } } } System.out.println("weight == " + weight); } else if (packagingTitle.trim().equals("Package Size:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { width = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); height = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(1)); length = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(2)); } catch (Exception e) { width = 1.0; height = 1.0; length = 1.0; } } } System.out.println("width == " + width); System.out.println("height == " + height); System.out.println("length == " + length); } } //=================== Packaging block END==================// //=================== Category block START==================// detailMain = doc.select("div.ui-breadcrumb div.container a"); Long productCategoryId = 0L; String parentPath = ""; String thisCategory = detailMain.last().text().trim(); System.out.println("thisCategory == " + thisCategory); Category parentCategory = new Category(); parentCategory.setId(0L); parentCategory.setParentPath(""); for (Element element : detailMain) { String newCategory; newCategory = element.text().trim(); System.out.println("newCategory======" + newCategory); if (newCategory.equals("Home") || newCategory.equals("All Categories")) { } else { Category category = categoryDao.getCategoryByName(newCategory); if (category != null) { if (category.getName().equals(thisCategory)) { productCategoryId = category.getId(); parentPath = category.getParentPath(); } parentCategory = category; } else { category = new Category(); category.setId(null); category.setName(newCategory); category.setParentId(parentCategory.getId()); category.setParentPath(parentCategory.getParentPath() + parentCategory.getId() + "="); category.setQuantity(0); category.setImgUrl("-"); category.setDisplayText(newCategory); Category category2 = categoryDao.save(category); if (category.getName().equals(thisCategory)) { productCategoryId = category2.getId(); parentPath = category2.getParentPath(); } parentCategory = category2; } } } //=================== Category block END==================// //=============== Specifications block START==============// detailMain = doc.select(".product-property-list .property-item"); String specifications = ""; for (Element element : detailMain) { specifications = specifications + element.select(".propery-title").get(0).text().replace(",", "/") .replace(":", "-") + ":" + element.select(".propery-des").get(0).text() .replace(",", "/").replace(":", "-") + ",";//TODO:, check } //=============== Specifications Block END==============// //=============== Shipping Time Block START==============// String shippingTime = ""; detailMain = doc.select(".shipping-days[data-role='delivery-days']"); System.out.println("value detailMain" + detailMain.toString()); shippingTime = detailMain.text(); //=============== Shipping Time Block END==============// //=============== Shipping Cost Block START==============// detailMain = doc.select(".logistics-cost"); value = detailMain.text(); if (!value.equalsIgnoreCase("Free Shipping")) { // f = 0.00; } else { // f = Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1")); } //=============== Shipping Cost Block END==============// //=================Product save 1st START==============// testProduct.setCategoryId(productCategoryId); testProduct.setLastUpdate(new Date()); testProduct.setParentPath(parentPath); testProduct.setImgurl("-"); testProduct.setProperties("-"); testProduct.setProductWidth(width); testProduct.setProductLength(length); testProduct.setProductWeight(weight); testProduct.setProductHeight(height); testProduct.setShippingRate(0.0); testProduct.setShippingTime("45"); testProduct.setSpecifications(specifications); savedTestProduct = productDao.save(testProduct); //====================Product save 1st END==============// //========= Property, Property Value, Property Product Map Block START ========// double discountPrice = 0.0; double actualPrice = 0.0; double markupPrice = 0.0; String id = ""; String allProperties = ""; //------------------------Read Color css START---------------------// specifics = doc.select("#j-product-info-sku dl.p-property-item"); Elements cssdetailMain = doc.select("link[href]"); Document cssdoc = new Document(""); System.out.println( "====================================================cssdetailMain" + cssdetailMain.size()); for (Element element : cssdetailMain) { String cssurl = element.attr("abs:href"); if (cssurl.contains("??main-detail")) { try { cssdoc = Jsoup.connect(cssurl).get(); } catch (IOException ex) { } break; } } //-----------------------Read Color css END--------------------------// //-----------Product Property, Property Value START--------// Map<String, ProductPropertyvalues> propertyValuesMap = new HashMap<>(); if (!specifics.isEmpty()) { ProductProperties testPorperties; ProductProperties saveTestPorperties; ProductPropertyvalues testPropertyValues; for (Element specific : specifics) { System.out.println("head ==== " + specific.select("dt").text()); testPorperties = productPropertiesDao .loadByName(specific.select("dt").text()); if (testPorperties == null) { testPorperties = new ProductProperties(); testPorperties.setPropertyName(specific.select("dt").text()); saveTestPorperties = productPropertiesDao.save(testPorperties); } else { saveTestPorperties = testPorperties; } allProperties = allProperties + saveTestPorperties.getId().toString() + "-"; detailSub = specific.select("dd ul li"); String valu = "-"; for (Element element : detailSub) { testPropertyValues = new ProductPropertyvalues(); id = element.select("a[data-sku-id]").attr("data-sku-id").trim(); testPropertyValues.setRefId(id); if (element.hasClass("item-sku-image")) { valu = element.select("a img[src]").get(0).absUrl("src") .split(".jpg")[0] + ".jpg"; String title = element.select("a img").get(0).attr("title"); String imgUrl = GoogleBucketFileUploader .uploadProductImage(valu, savedTestProduct.getId()); valu = "<img src='" + imgUrl + "' title='" + title + "' style='height:40px; width:40px;'/>"; } else if (element.hasClass("item-sku-color")) { String style = cssdoc.html().split("sku-color-" + id)[1] .split("}")[0].substring(1); valu = "<span style='" + style + "' ; height:40px; width:40px; display:block;'></span>"; } else { valu = element.select("a span").toString(); } System.out.println("valu === " + valu); testPropertyValues.setProductId(savedTestProduct.getId()); testPropertyValues.setPropertyId(saveTestPorperties.getId()); testPropertyValues.setValueName(valu); propertyValuesMap.put(id, productPropertyvaluesDao.save(testPropertyValues)); } } savedTestProduct.setProperties(allProperties); } //-----------Product Property, Property Value END--------// //----------------------Read json START------------------// List<AxpProductDto> axpProductDtos = new ArrayList<>(); Elements scripts = doc.select("script"); // Get the script part for (Element script : scripts) { if (script.html().contains("var skuProducts=")) { String jsonData = ""; jsonData = script.html().split("var skuProducts=")[1] .split("var GaData")[0].trim(); jsonData = jsonData.substring(0, jsonData.length() - 1); Gson gsonObj = new Gson(); axpProductDtos = Arrays .asList(gsonObj.fromJson(jsonData, AxpProductDto[].class)); break; } } //----------------------Read json END------------------// //-------------Product Properties Map START------------// for (AxpProductDto thisAxpProductDto : axpProductDtos) { SkuVal skuVal = thisAxpProductDto.getSkuVal(); if (skuVal.getActSkuCalPrice() != null) { value = skuVal.getActSkuCalPrice().trim(); discountPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = discountPrice * 0.15 + 100; discountPrice = Math.ceil((discountPrice + markupPrice) / 10) * 10; actualPrice = Math.round(actualPrice + markupPrice); } else { discountPrice = 0.0; value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = actualPrice * 0.15 + 100; discountPrice = Math.round(actualPrice + markupPrice); actualPrice = Math.round(actualPrice + markupPrice); } ProductPropertiesMap productPropertyMap = new ProductPropertiesMap(); String myPropValueIds = ""; if (thisAxpProductDto.getSkuAttr() != null) { String[] skuPropIds = thisAxpProductDto.getSkuPropIds().split(","); for (String skuPropId : skuPropIds) { myPropValueIds = myPropValueIds + propertyValuesMap.get(skuPropId).getId().toString() + "_"; } productPropertyMap.setPropertyvalueComposition(myPropValueIds); } else { productPropertyMap.setPropertyvalueComposition("_"); } productPropertyMap.setDiscount(discountPrice); productPropertyMap.setPrice(actualPrice); productPropertyMap.setProductId(savedTestProduct); productPropertyMap.setQuantity(5l); productPropertiesMapDao.save(productPropertyMap); } //-------------Product Properties Map START------------// //========= Property, Property Value, Property Product Map Block END ========// //============= Multiple Image Block START =============// detailMain = doc.select("ul.image-thumb-list span.img-thumb-item img[src]"); int flg = 0; String imgUrl = ""; for (Element element : detailMain) { imgUrl = GoogleBucketFileUploader.uploadProductImage( element.absUrl("src").split(".jpg")[0] + ".jpg", savedTestProduct.getId()); if (flg == 0) { flg++; savedTestProduct.setImgurl(imgUrl); } else { ProductImg productImg = new ProductImg(); productImg.setId(null); productImg.setImgUrl(imgUrl); productImg.setProductId(savedTestProduct.getId()); productImgDao.save(productImg); } } //============= Multiple Image Block END =============// //=================Product save final START==============// if (productDao.save(savedTestProduct) != null) { temtproductlinklist.setStatus(1);// temtproductlinklistDao.save(temtproductlinklist); status = "Success"; } //=================Product save final START==============// } else { temtproductlinklist.setStatus(2);// temtproductlinklistDao.save(temtproductlinklist); status = "criteria mismatch"; } } else { status = "Page not found"; } } catch (Exception ex) { System.out.println( "=============================================================Exception1" + ex); temtproductlinklist.setStatus(4);// temtproductlinklistDao.save(temtproductlinklist); System.out.println("Exception === " + ex); status = "Failure"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error in sub process.....\n Link Id: " + statusBean.getId() + "\n Started on" + startDate, ex); } } else { temtproductlinklist.setStatus(3);// temtproductlinklistDao.save(temtproductlinklist); status = "Product exsist"; } } // String body = "Id: " + temtproductlinklist.getId() + "<br/> Status: " + status; // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Product captured", body, "subhendu.sett@pkweb.in"); statusBean.setStatus(status); } System.out.println("=============================================================status" + status); } catch (Exception e) { System.out.println("=============================================================Exception2" + e); isSuccess = false; String body = "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error.....<br/> Started on" + startDate + "<br/>"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body, e); // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Stopped store product details", body + e.getLocalizedMessage(), "subhendu.sett@pkweb.in"); } if (isSuccess) { String body = "(==I==)DATE: " + new Date().toString() + "Store product details in temp product table end.....<br/> Started on" + startDate; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body); /*ObjectMapper mapper = new ObjectMapper(); try { MailSender.sendEmail("krisanu.nandi@pkweb.in", "Completed store product details", body + "=============<br/><br/>" + mapper.writeValueAsString(statusBeans), "subhendu.sett@pkweb.in"); } catch (JsonProcessingException ex) { Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, null, ex); }*/ } // return statusBeans; System.out.println("=============================================================end"); }