com.waku.mmdataextract.SalesIssues3.java Source code

Java tutorial

Introduction

Here is the source code for com.waku.mmdataextract.SalesIssues3.java

Source

/*
 * EasyDone.java
 * Created on 2011-2-24; Project to Colt2010; $Id: SalesIssues3.java 309 2013-04-25 16:38:44Z tristan $
 * 
 * Copyright (c) 2011, Xu Brothers and/or its affiliates. All rights reserved.
 * Xu Brothers PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
 */

package com.waku.mmdataextract;

import java.io.FileWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

/**
 * @author Jin
 */

public class SalesIssues3 {

    public static void main(String[] args) throws Exception {
        FileWriter fw = new FileWriter("Output.csv");
        Map<String, String> cityIdNameMap = new HashMap<String, String>();
        Map<String, Set<SalesIssue>> citySalesIssueMap = new HashMap<String, Set<SalesIssue>>();

        String cityMapString = "116=??, 125=, 117=, 127=, 114=, 115=?, 112=?, 122=, 113=, 110=, 123=, 124=?, 230=, 272=?, 129=, 118=, 119=, 109=, 107=, 106=, 105=, 104=";
        String[] cityMaps = cityMapString.split(", ");
        for (String cityMap : cityMaps) {
            String[] cityIdName = cityMap.split("=");
            cityIdNameMap.put(cityIdName[0], cityIdName[1]);
        }

        String salesIssues = "113=[310,471:?--20110101, 311,310:--20101116, 156,133:?--?20108, 154,132:?3G--20108, 312,469:?--20110101, 149,171:??--20100907, 217,235:??--HZ20101012, -1:?, 311,470:--20110101, 312,311:?--20101116, 156,231:?--HZ20101012, 216,234:?--HZ20101012, 310,309:?--20101116, 215,472:?--20110101, 215,233:?--HZ20101012, 149,129:??--20108, 156,170:?--20100907, 149,232:??--HZ20101012, 154,169:?3G--20100907, 154,212:?3G--HZ20101012], 107=[250,449:?--20110101, 250,369:?--20101215, 250,251:?--20101018, -1:?, 190,190:?--20100825, 249,249:?--20101018, 229,350:--20101215, 189,189:?8+1--20100825, 249,349:?--20101215, 229,230:--20101015, 249,450:?--20110101, 229,451:--20110101], 106=[209,209:?--20101010, 210,210:?--20101010, 210,510:?--20110101, 211,511:--20110101, 209,390:?--20101216, 211,211:--20101010, 209,509:?--20110101, 210,389:?--20101216, 211,391:--20101216, -1:?], 105=[46,46:5--0507, 45,45:?--?, 44,44:?9?--20100507, -1:?], 104=[-1:?, 66,64:?--20100624, 23,107:?0--20100721, 66,108:?--20100721, 23,23:?0--201000624, 68,65:0--2G20100624, 271,270:--20101020, 290,430:--20110101, 290,289:--20101109, 269,269:?--20101020, 68,149:0--20100830, 66,151:?--20100830, 273,271:--20101020, 23,150:?0--20100830, 269,429:?--20110101], 119=[104,329:?--20101209, 291,409:--20101223, 291,489:--20110101, 124,229:?--20101014, -1:?, 124,490:?--20110101, 124,104:?--20100714, 291,290:--20101109, 104,84:?--20100708";
        String[] forCity = salesIssues.split("], ");
        for (String eachCity : forCity) {
            String cityId = eachCity.substring(0, eachCity.indexOf("="));
            String salesIssuesForCity = eachCity.substring(eachCity.indexOf("=") + 2);
            Set<SalesIssue> salesIssueSet = new HashSet<SalesIssue>();
            for (String eachSales : salesIssuesForCity.split(", ")) {
                String[] a = eachSales.split(":");
                if (!a[0].equals("-1")) {
                    SalesIssue salesIssue = new SalesIssue(a[0], a[1]);
                    salesIssueSet.add(salesIssue);
                }
            }
            citySalesIssueMap.put(cityId, salesIssueSet);
        }

        String baseUrl = "http://shouji.gd.chinamobile.com/gdmobile/productDetail/operatorssalesSearchAction.do?type=1&brandId=0&productId=0&provinceId=1";
        WebDriver driver = new ChromeDriver();
        for (String cityId : citySalesIssueMap.keySet()) {
            for (SalesIssue salesIssue : citySalesIssueMap.get(cityId)) {
                String finalUrl = baseUrl + "&cityId=" + cityId + "&salesissue=" + salesIssue.id;
                driver.get(finalUrl);
                System.out.println("Go to url => " + finalUrl);
                fw.write("Go to url => " + finalUrl + "\n");
                System.out
                        .println("Province,,City," + cityIdNameMap.get(cityId) + ",Sales," + salesIssue.name);
                fw.write("Province,,City," + cityIdNameMap.get(cityId) + ",Sales," + salesIssue.name + "\n");
                List<WebElement> tables = driver.findElements(By.tagName("table"));
                // Second table
                WebElement mainTable = tables.get(1);
                List<WebElement> trs = mainTable.findElements(By.tagName("tr"));
                Boolean firstOne = true;
                int cols = 0;
                for (WebElement tr : trs) {
                    try {
                        if (firstOne) {
                            StringBuilder header = new StringBuilder();
                            firstOne = false;
                            List<WebElement> tds = tr.findElements(By.tagName("td"));
                            for (WebElement td : tds) {
                                try {
                                    String s = td.findElement(By.tagName("strong")).getText();
                                    header.append(s + ",");
                                } catch (NoSuchElementException e) {
                                    header.append("ImageURL,");
                                }
                            }
                            System.out.println(header.toString());
                            fw.write(header.toString() + "\n");
                            cols = header.toString().split(",").length;
                        } else {
                            List<WebElement> tds = tr.findElements(By.tagName("td"));
                            if (tds.size() == cols) {
                                StringBuilder line = new StringBuilder();
                                for (WebElement td : tds) {
                                    try {
                                        String s = "http://shouji.gd.chinamobile.com"
                                                + td.findElement(By.tagName("img")).getAttribute("src");
                                        line.append(s + ",");
                                    } catch (NoSuchElementException e) {
                                        line.append(td.getText() + ",");
                                    }
                                }
                                System.out.println(line);
                                fw.write(line.toString() + "\n");
                            } else {
                                StringBuilder line = new StringBuilder();
                                for (int i = 0; i < (cols - tds.size()); i++) {
                                    line.append(",");
                                }
                                for (WebElement td : tds) {
                                    line.append(td.getText() + ",");
                                }
                                System.out.println(line);
                                fw.write(line.toString() + "\n");
                            }
                        }
                    } catch (NoSuchElementException e) {
                        System.out.println("One page end!");
                        fw.write("One page end!" + "\n");
                    }
                }

                // Third table
                WebElement lastTable = tables.get(1);
                List<WebElement> span = lastTable.findElements(By.xpath("//span[@class='font_b2']"));
                int pageNumber = Integer.parseInt(span.get(1).getText());
                if (pageNumber > 1) {
                    for (int i = 2; i <= pageNumber; i++) {
                        goToURL(fw, driver, finalUrl + "&currentPage=" + i);
                    }
                }
            }
        }
        fw.close();
    }

    private static void goToURL(FileWriter fw, WebDriver driver, String finalUrl) throws Exception {
        driver.get(finalUrl);
        List<WebElement> tables = driver.findElements(By.tagName("table"));
        // Second table
        WebElement mainTable = tables.get(1);
        List<WebElement> trs = mainTable.findElements(By.tagName("tr"));
        Boolean firstOne = true;
        int cols = 0;
        for (WebElement tr : trs) {
            try {
                if (firstOne) {
                    StringBuilder header = new StringBuilder();
                    firstOne = false;
                    List<WebElement> tds = tr.findElements(By.tagName("td"));
                    for (WebElement td : tds) {
                        try {
                            String s = td.findElement(By.tagName("strong")).getText();
                            header.append(s + ",");
                        } catch (NoSuchElementException e) {
                            header.append("ImageURL,");
                        }
                    }
                    System.out.println(header.toString());
                    fw.write(header + "\n");
                    cols = header.toString().split(",").length;
                } else {
                    List<WebElement> tds = tr.findElements(By.tagName("td"));
                    if (tds.size() == cols) {
                        StringBuilder line = new StringBuilder();
                        for (WebElement td : tds) {
                            try {
                                String s = "http://shouji.gd.chinamobile.com"
                                        + td.findElement(By.tagName("img")).getAttribute("src");
                                line.append(s + ",");
                            } catch (NoSuchElementException e) {
                                line.append(td.getText() + ",");
                            }
                        }
                        System.out.println(line);
                        fw.write(line.toString() + "\n");
                    } else {
                        StringBuilder line = new StringBuilder();
                        for (int i = 0; i < (cols - tds.size()); i++) {
                            line.append(",");
                        }
                        for (WebElement td : tds) {
                            line.append(td.getText() + ",");
                        }
                        System.out.println(line);
                        fw.write(line.toString() + "\n");
                    }
                }
            } catch (NoSuchElementException e) {
                System.out.println("One page end!");
                fw.write("One page end!".toString() + "\n");
            }
        }
    }
}

// A class to hold Sales issue id and name
class SalesIssue {

    @Override
    public String toString() {
        return id + ":" + name;
    }

    public SalesIssue(String id, String name) {
        this.id = id;
        this.name = name;
    }

    String id;
    String name;
}