Java tutorial
/* * EasyDone.java * Created on 2011-2-24; Project to Colt2010; $Id: SalesIssues3.java 309 2013-04-25 16:38:44Z tristan $ * * Copyright (c) 2011, Xu Brothers and/or its affiliates. All rights reserved. * Xu Brothers PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. */ package com.waku.mmdataextract; import java.io.FileWriter; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import org.openqa.selenium.By; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.chrome.ChromeDriver; /** * @author Jin */ public class SalesIssues3 { public static void main(String[] args) throws Exception { FileWriter fw = new FileWriter("Output.csv"); Map<String, String> cityIdNameMap = new HashMap<String, String>(); Map<String, Set<SalesIssue>> citySalesIssueMap = new HashMap<String, Set<SalesIssue>>(); String cityMapString = "116=??, 125=, 117=, 127=, 114=, 115=?, 112=?, 122=, 113=, 110=, 123=, 124=?, 230=, 272=?, 129=, 118=, 119=, 109=, 107=, 106=, 105=, 104="; String[] cityMaps = cityMapString.split(", "); for (String cityMap : cityMaps) { String[] cityIdName = cityMap.split("="); cityIdNameMap.put(cityIdName[0], cityIdName[1]); } String salesIssues = "113=[310,471:?--20110101, 311,310:--20101116, 156,133:?--?20108, 154,132:?3G--20108, 312,469:?--20110101, 149,171:??--20100907, 217,235:??--HZ20101012, -1:?, 311,470:--20110101, 312,311:?--20101116, 156,231:?--HZ20101012, 216,234:?--HZ20101012, 310,309:?--20101116, 215,472:?--20110101, 215,233:?--HZ20101012, 149,129:??--20108, 156,170:?--20100907, 149,232:??--HZ20101012, 154,169:?3G--20100907, 154,212:?3G--HZ20101012], 107=[250,449:?--20110101, 250,369:?--20101215, 250,251:?--20101018, -1:?, 190,190:?--20100825, 249,249:?--20101018, 229,350:--20101215, 189,189:?8+1--20100825, 249,349:?--20101215, 229,230:--20101015, 249,450:?--20110101, 229,451:--20110101], 106=[209,209:?--20101010, 210,210:?--20101010, 210,510:?--20110101, 211,511:--20110101, 209,390:?--20101216, 211,211:--20101010, 209,509:?--20110101, 210,389:?--20101216, 211,391:--20101216, -1:?], 105=[46,46:5--0507, 45,45:?--?, 44,44:?9?--20100507, -1:?], 104=[-1:?, 66,64:?--20100624, 23,107:?0--20100721, 66,108:?--20100721, 23,23:?0--201000624, 68,65:0--2G20100624, 271,270:--20101020, 290,430:--20110101, 290,289:--20101109, 269,269:?--20101020, 68,149:0--20100830, 66,151:?--20100830, 273,271:--20101020, 23,150:?0--20100830, 269,429:?--20110101], 119=[104,329:?--20101209, 291,409:--20101223, 291,489:--20110101, 124,229:?--20101014, -1:?, 124,490:?--20110101, 124,104:?--20100714, 291,290:--20101109, 104,84:?--20100708"; String[] forCity = salesIssues.split("], "); for (String eachCity : forCity) { String cityId = eachCity.substring(0, eachCity.indexOf("=")); String salesIssuesForCity = eachCity.substring(eachCity.indexOf("=") + 2); Set<SalesIssue> salesIssueSet = new HashSet<SalesIssue>(); for (String eachSales : salesIssuesForCity.split(", ")) { String[] a = eachSales.split(":"); if (!a[0].equals("-1")) { SalesIssue salesIssue = new SalesIssue(a[0], a[1]); salesIssueSet.add(salesIssue); } } citySalesIssueMap.put(cityId, salesIssueSet); } String baseUrl = "http://shouji.gd.chinamobile.com/gdmobile/productDetail/operatorssalesSearchAction.do?type=1&brandId=0&productId=0&provinceId=1"; WebDriver driver = new ChromeDriver(); for (String cityId : citySalesIssueMap.keySet()) { for (SalesIssue salesIssue : citySalesIssueMap.get(cityId)) { String finalUrl = baseUrl + "&cityId=" + cityId + "&salesissue=" + salesIssue.id; driver.get(finalUrl); System.out.println("Go to url => " + finalUrl); fw.write("Go to url => " + finalUrl + "\n"); System.out .println("Province,,City," + cityIdNameMap.get(cityId) + ",Sales," + salesIssue.name); fw.write("Province,,City," + cityIdNameMap.get(cityId) + ",Sales," + salesIssue.name + "\n"); List<WebElement> tables = driver.findElements(By.tagName("table")); // Second table WebElement mainTable = tables.get(1); List<WebElement> trs = mainTable.findElements(By.tagName("tr")); Boolean firstOne = true; int cols = 0; for (WebElement tr : trs) { try { if (firstOne) { StringBuilder header = new StringBuilder(); firstOne = false; List<WebElement> tds = tr.findElements(By.tagName("td")); for (WebElement td : tds) { try { String s = td.findElement(By.tagName("strong")).getText(); header.append(s + ","); } catch (NoSuchElementException e) { header.append("ImageURL,"); } } System.out.println(header.toString()); fw.write(header.toString() + "\n"); cols = header.toString().split(",").length; } else { List<WebElement> tds = tr.findElements(By.tagName("td")); if (tds.size() == cols) { StringBuilder line = new StringBuilder(); for (WebElement td : tds) { try { String s = "http://shouji.gd.chinamobile.com" + td.findElement(By.tagName("img")).getAttribute("src"); line.append(s + ","); } catch (NoSuchElementException e) { line.append(td.getText() + ","); } } System.out.println(line); fw.write(line.toString() + "\n"); } else { StringBuilder line = new StringBuilder(); for (int i = 0; i < (cols - tds.size()); i++) { line.append(","); } for (WebElement td : tds) { line.append(td.getText() + ","); } System.out.println(line); fw.write(line.toString() + "\n"); } } } catch (NoSuchElementException e) { System.out.println("One page end!"); fw.write("One page end!" + "\n"); } } // Third table WebElement lastTable = tables.get(1); List<WebElement> span = lastTable.findElements(By.xpath("//span[@class='font_b2']")); int pageNumber = Integer.parseInt(span.get(1).getText()); if (pageNumber > 1) { for (int i = 2; i <= pageNumber; i++) { goToURL(fw, driver, finalUrl + "¤tPage=" + i); } } } } fw.close(); } private static void goToURL(FileWriter fw, WebDriver driver, String finalUrl) throws Exception { driver.get(finalUrl); List<WebElement> tables = driver.findElements(By.tagName("table")); // Second table WebElement mainTable = tables.get(1); List<WebElement> trs = mainTable.findElements(By.tagName("tr")); Boolean firstOne = true; int cols = 0; for (WebElement tr : trs) { try { if (firstOne) { StringBuilder header = new StringBuilder(); firstOne = false; List<WebElement> tds = tr.findElements(By.tagName("td")); for (WebElement td : tds) { try { String s = td.findElement(By.tagName("strong")).getText(); header.append(s + ","); } catch (NoSuchElementException e) { header.append("ImageURL,"); } } System.out.println(header.toString()); fw.write(header + "\n"); cols = header.toString().split(",").length; } else { List<WebElement> tds = tr.findElements(By.tagName("td")); if (tds.size() == cols) { StringBuilder line = new StringBuilder(); for (WebElement td : tds) { try { String s = "http://shouji.gd.chinamobile.com" + td.findElement(By.tagName("img")).getAttribute("src"); line.append(s + ","); } catch (NoSuchElementException e) { line.append(td.getText() + ","); } } System.out.println(line); fw.write(line.toString() + "\n"); } else { StringBuilder line = new StringBuilder(); for (int i = 0; i < (cols - tds.size()); i++) { line.append(","); } for (WebElement td : tds) { line.append(td.getText() + ","); } System.out.println(line); fw.write(line.toString() + "\n"); } } } catch (NoSuchElementException e) { System.out.println("One page end!"); fw.write("One page end!".toString() + "\n"); } } } } // A class to hold Sales issue id and name class SalesIssue { @Override public String toString() { return id + ":" + name; } public SalesIssue(String id, String name) { this.id = id; this.name = name; } String id; String name; }