Java tutorial
/******************************************************************************* * Copyright 2015 htd0324@gmail.com * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. ******************************************************************************/ package com.laudandjolynn.mytv; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.laudandjolynn.mytv.datasource.DataSourceManager; import com.laudandjolynn.mytv.event.CrawlEvent; import com.laudandjolynn.mytv.event.CrawlEventListener; import com.laudandjolynn.mytv.event.CrawlEventListenerAdapter; import com.laudandjolynn.mytv.event.TvStationFoundEvent; import com.laudandjolynn.mytv.exception.MyTvException; import com.laudandjolynn.mytv.model.MyTv; import com.laudandjolynn.mytv.model.TvStation; import com.laudandjolynn.mytv.proxy.ConfigProxy; import com.laudandjolynn.mytv.proxy.MyTvProxyManager; import com.laudandjolynn.mytv.service.TvService; import com.laudandjolynn.mytv.service.TvServiceImpl; import com.laudandjolynn.mytv.utils.Constant; import com.laudandjolynn.mytv.utils.DateUtils; import com.laudandjolynn.mytv.utils.MemoryCache; /** * @author: Laud * @email: htd0324@gmail.com * @date: 2015325 ?9:12:56 * @copyright: www.laudandjolynn.com */ public class Main { private final static Logger logger = LoggerFactory.getLogger(Main.class); public static void main(String[] args) { // ???? startService(); com.laudandjolynn.mytv.Server hessian = new HessianServer(); try { hessian.start(); } catch (Exception e) { logger.error(e.getMessage(), e); throw new MyTvException(e); } com.laudandjolynn.mytv.Server rmi = new RmiServer(); try { rmi.start(); } catch (Exception e) { logger.error(e.getMessage(), e); throw new MyTvException(e); } logger.info("My TV Program Table Crawler is running."); } /** * ? */ private static void startService() { logger.info("start My TV Program Table Crawler."); // ? MyTvData data = new MyTvData(); // ?? initDb(data); // ??? initDbData0(data); initDbData(data); } @SuppressWarnings("unchecked") private static List<String> loadSql() { Object object = DataSourceManager.DATA_SOURCE_PROP.get(DataSourceManager.RES_KEY_DB_SQL_LIST); if (object instanceof List) { return (List<String>) object; } throw new MyTvException("error occur while trying to init db."); } /** * ?? */ private static void initDb(MyTvData data) { if (data.isDbInited()) { logger.debug("db have already init."); return; } File myTvDataFilePath = new File(Constant.MY_TV_DATA_FILE_PATH); Connection conn = null; Statement stmt = null; try { conn = DataSourceManager.getConnection(); conn.setAutoCommit(false); stmt = conn.createStatement(); List<String> sqlList = loadSql(); for (String sql : sqlList) { stmt.addBatch(sql); logger.info("execute sql: " + sql); } stmt.executeBatch(); conn.commit(); DataSourceManager.DATA_SOURCE_PROP.remove(DataSourceManager.RES_KEY_DB_SQL_LIST); data.writeData(null, Constant.XML_TAG_DB, "true"); } catch (SQLException e) { if (conn != null) { try { conn.rollback(); } catch (SQLException e1) { throw new MyTvException("error occur while rollback transaction.", e); } } myTvDataFilePath.deleteOnExit(); throw new MyTvException("error occur while execute sql on db.", e); } finally { if (stmt != null) { try { stmt.close(); } catch (SQLException e) { throw new MyTvException("error occur while close statement.", e); } } if (conn != null) { try { conn.close(); } catch (SQLException e) { throw new MyTvException("error occur while close sqlite connection.", e); } } } } /** * ??? */ private static void initDbData0(MyTvData data) { if (data.isDataInited()) { logger.info("init data had insert into db."); return; } Properties tvStationProp = new Properties(); try { tvStationProp.load(Main.class.getResourceAsStream("/" + Constant.TV_STATION_INIT_DATA_FILE_NAME)); } catch (IOException e) { throw new MyTvException( "error occur while load property file: " + Constant.TV_STATION_INIT_DATA_FILE_NAME, e); } Collection<Object> values = tvStationProp.values(); List<String> insertSqlList = new ArrayList<String>(values.size()); String insertSql = "insert into my_tv (stationName,displayName,classify,channel,sequence)"; for (Object value : values) { insertSqlList.add(insertSql + " values (" + value.toString() + ")"); } Connection conn = null; Statement stmt = null; try { conn = DataSourceManager.getConnection(); conn.setAutoCommit(false); stmt = conn.createStatement(); for (String sql : insertSqlList) { stmt.addBatch(sql); logger.info("execute sql: " + sql); } stmt.executeBatch(); conn.commit(); data.writeData(null, Constant.XML_TAG_DATA, "true"); } catch (SQLException e) { if (conn != null) { try { conn.rollback(); } catch (SQLException e1) { throw new MyTvException("error occur while rollback transaction.", e1); } } throw new MyTvException("error occur while execute sql on db.", e); } finally { if (stmt != null) { try { stmt.close(); } catch (SQLException e) { throw new MyTvException("error occur while close statement.", e); } } if (conn != null) { try { conn.close(); } catch (SQLException e) { throw new MyTvException("error occur while close sqlite connection.", e); } } } } private static void makeCache(TvService tvService) { List<TvStation> stationList = tvService.getAllCrawlableStation(); if (stationList != null) { MemoryCache.getInstance().addCache(stationList); } List<MyTv> myTvList = tvService.getMyTv(); if (myTvList != null) { MemoryCache.getInstance().addMyTvCache(myTvList); } } /** * ?? */ private static void initDbData(final MyTvData data) { final TvService tvService = new TvServiceImpl(); makeCache(tvService); // ?? ExecutorService executorService = Executors.newSingleThreadExecutor( new BasicThreadFactory.Builder().namingPattern("Mytv_Crawl_Task_%d").build()); executorService.execute(new Runnable() { @Override public void run() { runCrawlTask(data, tvService); } }); executorService.shutdown(); // ?? logger.info("create everyday crawl task."); createEverydayCron(data, tvService); } /** * ? * * @param data */ private static void runCrawlTask(final MyTvData data, final TvService tvService) { CrawlEventListener listener = null; final String today = DateUtils.today(); final ExecutorService executorService = Executors.newFixedThreadPool(Constant.CPU_PROCESSOR_NUM, new BasicThreadFactory.Builder().namingPattern("Mytv_Crawl_Program_Table_%d").build()); if (!data.isProgramCrawlerInited()) { listener = new CrawlEventListenerAdapter() { @Override public void itemFound(CrawlEvent event) { if (event instanceof TvStationFoundEvent) { final TvStation item = (TvStation) ((TvStationFoundEvent) event).getItem(); if (!tvService.isInMyTv(item) || CrawlAction.getIntance().isInQuerying(item, today)) { return; } executorService.execute(new Runnable() { @Override public void run() { CrawlAction.getIntance().queryProgramTable(item, today); } }); } } }; } // ??? logger.info("It is trying to find some proxyies."); MyTvProxyManager.getInstance().prepareProxies(new ConfigProxy()); logger.info("found " + MyTvProxyManager.getInstance().getProxySize() + " proxies."); if (!data.isStationCrawlerInited()) { // ? tvService.crawlAllTvStation(listener); // ?? String[] weeks = DateUtils.getWeek(new Date(), "yyyy-MM-dd"); List<TvStation> stationList = tvService.getDisplayedTvStation(); for (String date : weeks) { if (date.compareTo(today) >= 1) { crawlAllProgramTable(stationList, executorService, date, tvService); } } executorService.shutdown(); data.writeData(null, Constant.XML_TAG_STATION, "true"); data.writeData(null, Constant.XML_TAG_PROGRAM, "true"); } } /** * ? * * @param data * @param tvService */ private static void createEverydayCron(final MyTvData data, final TvService tvService) { ScheduledExecutorService scheduled = new ScheduledThreadPoolExecutor(3, new BasicThreadFactory.Builder().namingPattern("Mytv_Scheduled_Task").build()); Date today = new Date(); String nextWeek = DateUtils.date2String(DateUtils.nextWeek(today), "yyyy-MM-dd 00:01:00"); long crawlTaskInitDelay = (DateUtils.string2Date(nextWeek).getTime() - today.getTime()) / 1000; logger.info("cron crawler task will be automatic start after " + crawlTaskInitDelay + " seconds at " + nextWeek); scheduled.scheduleWithFixedDelay(new Runnable() { @Override public void run() { Date[] weeks = DateUtils.getWeek(new Date()); logger.info("begin to crawl program table of " + Arrays.deepToString(weeks)); ExecutorService executorService = Executors.newFixedThreadPool(Constant.CPU_PROCESSOR_NUM, new BasicThreadFactory.Builder().namingPattern("Mytv_Schedule_Crawl_Program_Table_%d") .build()); List<TvStation> stationList = tvService.getDisplayedTvStation(); for (Date date : weeks) { crawlAllProgramTable(stationList, executorService, DateUtils.date2String(date, "yyyy-MM-dd"), tvService); } executorService.shutdown(); } }, crawlTaskInitDelay, 604860, TimeUnit.SECONDS); // ?? String nextDate = DateUtils.tommorow() + " 23:00:00"; long commonInitDelay = (DateUtils.string2Date(nextDate).getTime() - today.getTime()) / 1000; logger.info("cron refresh proxy task will be automatic start after " + commonInitDelay + " seconds at " + nextDate); scheduled.scheduleWithFixedDelay(new Runnable() { @Override public void run() { logger.info("begin to refresh proxies."); MyTvProxyManager.getInstance().refresh(); } }, commonInitDelay, 86400, TimeUnit.SECONDS); // logger.info("cron refresh cache task will be automatic start after " + commonInitDelay + " seconds at " + nextDate); scheduled.scheduleWithFixedDelay(new Runnable() { @Override public void run() { logger.info("begin to refresh caches."); makeCache(tvService); } }, commonInitDelay, 86400, TimeUnit.SECONDS); // scheduled????? // scheduled.shutdown(); } /** * ?? * * @param executorService * @param date * @param tvService */ private static void crawlAllProgramTable(List<TvStation> stationList, ExecutorService executorService, final String date, TvService tvService) { int size = stationList == null ? 0 : stationList.size(); for (int i = 0; i < size; i++) { final TvStation tvStation = stationList.get(i); executorService.execute(new Runnable() { @Override public void run() { CrawlAction.getIntance().queryProgramTable(tvStation, date); } }); } } }