public abstract class AbstractCrawler implements Crawler {
	protected List<CrawlEventListener> listeners = new ArrayList<CrawlEventListener>();

	@Override
	public void registerCrawlEventListener(CrawlEventListener listener) {
		this.listeners.add(listener);
	}

	@Override
	public void removeCrawlEventListener(CrawlEventListener listener) {
		this.listeners.remove(listener);
	}
}
 public class AllTvStationCrawlEndEvent extends CrawlEndEvent<List<TvStation>> {
	private static final long serialVersionUID = -3789839197425037620L;

	public AllTvStationCrawlEndEvent(Object source, List<TvStation> returnValue) {
		super(source, returnValue);
	}

}
 public interface CrawlEventListener {
	/**
	 * 抓取到电视台、电视节目对象的事件通知
	 * 
	 * @param event
	 */
	public void itemFound(CrawlEvent event);

	/**
	 * 抓取结束，即将返回结果
	 * 
	 * @param event
	 */
	public void crawlEnd(CrawlEvent event);
}
 public class ProgramTableCrawlEndEvent extends
		CrawlEndEvent<List<ProgramTable>> {
	private static final long serialVersionUID = 8031335081624277839L;
	private String stationName = null;
	private String date = null;

	public ProgramTableCrawlEndEvent(Object source,
			List<ProgramTable> returnValue, String stationName, String date) {
		super(source, returnValue);
		this.stationName = stationName;
		this.date = date;
	}

	public String getStationName() {
		return stationName;
	}

	public String getDate() {
		return date;
	}

}
 public class ProgramTableFoundEvent extends ItemFoundEvent<ProgramTable> {
	private static final long serialVersionUID = 734293143060487624L;

	public ProgramTableFoundEvent(Object source, ProgramTable item) {
		super(source, item);
	}

}
 public class TvStationFoundEvent extends ItemFoundEvent<TvStation> {
	private static final long serialVersionUID = -4729651908093312009L;

	public TvStationFoundEvent(Object source, TvStation item) {
		super(source, item);
	}

}
 public class ProgramTable {
	private long id;
	private String stationName;
	private String program;
	private String airDate;
	private String airTime;
	private int week;

	public long getId() {
		return id;
	}

	public void setId(long id) {
		this.id = id;
	}

	public String getStationName() {
		return stationName;
	}

	public void setStationName(String stationName) {
		this.stationName = stationName;
	}

	public String getProgram() {
		return program;
	}

	public void setProgram(String program) {
		this.program = program;
	}

	public String getAirDate() {
		return airDate;
	}

	public void setAirDate(String airDate) {
		this.airDate = airDate;
	}

	public String getAirTime() {
		return airTime;
	}

	public void setAirTime(String airTime) {
		this.airTime = airTime;
	}

	/**
	 * 1-7分别代表星期一，星期二，星期三，星期四，星期五，星期六，星期日
	 * 
	 * @return
	 */
	public int getWeek() {
		return week;
	}

	/**
	 * 1-7分别代表星期一，星期二，星期三，星期四，星期五，星期六，星期日
	 * 
	 * @param week
	 */
	public void setWeek(int week) {
		this.week = week;
	}

	@Override
	public String toString() {
		return "ProgramTable [id=" + id + ", stationName=" + stationName
				+ ", program=" + program + ", airDate=" + airDate
				+ ", airTime=" + airTime + ", week=" + week + "]";
	}

}
 public class TvStation {
	private int id;
	private String name;
	private String city;
	private String classify;
	private int sequence;

	public int getId() {
		return id;
	}

	public void setId(int id) {
		this.id = id;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public String getCity() {
		return city;
	}

	public void setCity(String city) {
		this.city = city;
	}

	public String getClassify() {
		return classify;
	}

	public void setClassify(String classify) {
		this.classify = classify;
	}

	public int getSequence() {
		return sequence;
	}

	public void setSequence(int sequence) {
		this.sequence = sequence;
	}

	@Override
	public int hashCode() {
		final int prime = 31;
		int result = 1;
		result = prime * result + ((city == null) ? 0 : city.hashCode());
		result = prime * result
				+ ((classify == null) ? 0 : classify.hashCode());
		result = prime * result + ((name == null) ? 0 : name.hashCode());
		return result;
	}

	@Override
	public boolean equals(Object obj) {
		if (this == obj)
			return true;
		if (obj == null)
			return false;
		if (getClass() != obj.getClass())
			return false;
		TvStation other = (TvStation) obj;
		if (city == null) {
			if (other.city != null)
				return false;
		} else if (!city.equals(other.city))
			return false;
		if (classify == null) {
			if (other.classify != null)
				return false;
		} else if (!classify.equals(other.classify))
			return false;
		if (name == null) {
			if (other.name != null)
				return false;
		} else if (!name.equals(other.name))
			return false;
		return true;
	}

	@Override
	public String toString() {
		return "TvStation [id=" + id + ", name=" + name + ", city=" + city
				+ ", classify=" + classify + ", sequence=" + sequence + "]";
	}

}
 public class Constant {
	// 应用名称
	public final static String APP_NAME = "mytv";
	// CPU核数
	public static final int CPU_PROCESSOR_NUM = Runtime.getRuntime()
			.availableProcessors();
	// 数据文件目录
	public final static String MY_TV_DATA_PATH = Config.getDataFilePath()
			+ "data" + File.separator;
	// 数据文件
	public final static String MY_TV_DATA_FILE_PATH = MY_TV_DATA_PATH
			+ "mytv.dat";
	// 电视节目表文件路径
	public final static String CRAWL_FILE_PATH = Constant.MY_TV_DATA_PATH
			+ "crawlfiles" + File.separator;

	public final static String XML_TAG_DB = "db";
	public final static String XML_TAG_DATA = "data";
	public final static String XML_TAG_STATION = "station";
	public final static String XML_TAG_PROGRAM = "program";

	public final static String TV_STATION_INIT_DATA_FILE_NAME = "tv_station.properties";
	public final static String TV_STATION_ALIAS_INIT_DATA_FILE_NAME = "tv_station_alias.properties";

	public final static String DOT = ".";
	public final static String COMMA = ",";
	public final static String UNDERLINE = "_";
	public final static String COLON = ":";

	static {
		File file = new File(MY_TV_DATA_PATH);
		if (!file.exists()) {
			file.mkdirs();
		}

		file = new File(CRAWL_FILE_PATH);
		if (!file.exists()) {
			file.mkdirs();
		}
	}

}
 public class DateUtils {
	private final static Logger logger = LoggerFactory
			.getLogger(DateUtils.class);
	// 默认时间格式
	public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd HH:mm:ss";

	/**
	 * 获取当天的日期字符串，yyyy-MM-dd
	 * 
	 * @return
	 */
	public static String today() {
		return DateUtils.date2String(Calendar.getInstance().getTime(),
				"yyyy-MM-dd");
	}

	/**
	 * 获取明天的日期字符串，yyyy-MM-dd
	 * 
	 * @return
	 */
	public static String tommorow() {
		Calendar calendar = Calendar.getInstance();
		calendar.add(Calendar.DAY_OF_MONTH, 1);
		return DateUtils.date2String(calendar.getTime(), "yyyy-MM-dd");
	}

	/**
	 * 字符串转为时间
	 * 
	 * @param date
	 * @param pattern
	 * @return
	 */
	public static Date string2Date(String date, String pattern) {
		SimpleDateFormat format = new SimpleDateFormat(pattern);
		try {
			return format.parse(date);
		} catch (ParseException e) {
			logger.error("invalid date value: " + date, e);
		}
		return null;
	}

	/**
	 * 字符串转为时间
	 * 
	 * @param date
	 * @return
	 */
	public static Date string2Date(String date) {
		return string2Date(date, DEFAULT_DATE_PATTERN);
	}

	/**
	 * 时间转为字符串
	 * 
	 * @param date
	 * @return yyyy-MM-dd HH:mm:ss格式
	 */
	public static String date2String(Date date) {
		SimpleDateFormat formatter = new SimpleDateFormat(DEFAULT_DATE_PATTERN);
		formatter.setTimeZone(TimeZone.getDefault());
		return formatter.format(date);
	}

	/**
	 * 时间转为字符串
	 * 
	 * @param date
	 * @param pattern
	 *            时间样式
	 * @return
	 */
	public static String date2String(Date date, String pattern) {
		SimpleDateFormat formatter = new SimpleDateFormat(pattern);
		formatter.setTimeZone(TimeZone.getDefault());
		return formatter.format(date);
	}

	/**
	 * 获取指定時間一周的日期列表
	 * 
	 * @param date
	 * @return
	 */
	public static Date[] getWeek(Date date) {
		Date[] dates = new Date[7];
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		calendar.setFirstDayOfWeek(Calendar.MONDAY);
		calendar.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY);
		for (int i = 0; i < 7; i++) {
			dates[i] = calendar.getTime();
			calendar.add(Calendar.DAY_OF_YEAR, 1);
		}
		return dates;
	}

	/**
	 * 获取指定时间的下一周头一天, 每周第一天为星期一
	 * 
	 * @param date
	 * @return
	 */
	public static Date nextWeek(Date date) {
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		calendar.setFirstDayOfWeek(Calendar.MONDAY);
		calendar.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY);
		calendar.add(Calendar.WEEK_OF_YEAR, 1);
		return calendar.getTime();
	}

	/**
	 * 获取指定时间一周的日期列表，每周第一天为星期一
	 * 
	 * @param date
	 * @param pattern
	 * @return
	 */
	public static String[] getWeek(Date date, String pattern) {
		Date[] dates = getWeek(date);
		String[] result = new String[dates.length];
		for (int i = 0; i < 7; i++) {
			result[i] = date2String(dates[i], pattern);
		}
		return result;
	}

	/**
	 * 给指定日期加/减年
	 * 
	 * @param date
	 * @param year
	 *            正数加，负数减
	 * @return
	 */
	public static Date addYear(Date date, int year) {
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		calendar.add(Calendar.YEAR, year);
		return calendar.getTime();
	}

	/**
	 * 给指定日期加/减指定月
	 * 
	 * @param date
	 * @param month
	 *            正数加，负数减
	 * @return
	 */
	public static Date addMonth(Date date, int month) {
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		calendar.add(Calendar.MONTH, month);
		return calendar.getTime();
	}

	/**
	 * 给指定日期加/减指定天数
	 * 
	 * @param date
	 * @param day
	 *            正数加，负数减
	 * @return
	 */
	public static Date addDay(Date date, int day) {
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		calendar.add(Calendar.DAY_OF_MONTH, day);
		return calendar.getTime();
	}

	/**
	 * 计算两个指定日期相差几天
	 * 
	 * @param arg0
	 * @param arg1
	 * @return 返回大减小的相差天数
	 */
	public static long differHowManyDays(Date arg0, Date arg1) {
		long d1 = arg0.getTime();
		long d2 = arg1.getTime();

		return Math.abs(d1 - d2) / (60 * 60 * 24 * 1000);
	}

	/**
	 * 取得一天的开始时间
	 * 
	 * @param date
	 * @return
	 */
	public static Date getStartOfTheDay(Date date) {
		String value = date2String(date, "yyyy-MM-dd 00:00:00");
		return string2Date(value);
	}

	/**
	 * 取得一天的结束时间
	 * 
	 * @param date
	 * @return
	 */
	public static Date getEndOfTheDay(Date date) {
		String value = date2String(date, "yyyy-MM-dd 23:59:59");
		return string2Date(value);
	}

	/**
	 * 取得指定月份的第一天
	 * 
	 * @param date
	 * @return
	 */
	public static Date getFirstDayOfTheMonth(Date date) {
		String value = date2String(date, "yyyy-MM-01 00:00:00");
		return string2Date(value);
	}

	/**
	 * 取得指定月份的最后一天
	 * 
	 * @param date
	 * @return
	 */
	public static Date getLastDayOfTheMonth(Date date) {
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(date);
		int lastDay = calendar.getActualMaximum(Calendar.DAY_OF_MONTH);
		calendar.set(calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH),
				lastDay, 0, 0, 0);
		return calendar.getTime();
	}

}
 public class MyTvUtils {
	private final static Logger logger = LoggerFactory
			.getLogger(MyTvUtils.class);

	/**
	 * 读入html文件
	 * 
	 * @param path
	 * @return
	 */
	public static String readAsHtml(String path) throws IOException {
		return new String(FileUtils.readWithNIO(path, "UTF-8"), "UTF-8");
	}

	/**
	 * 输出抓取数据到文件
	 * 
	 * @param dirName
	 *            目录名称，如日期，yyyy-MM-dd
	 * @param data
	 *            数据
	 * @param fileName
	 *            文件名
	 */
	public static void outputCrawlData(String dirName, String data,
			String fileName) {
		String crawlFileDir = Constant.CRAWL_FILE_PATH + dirName
				+ File.separator;
		File file = new File(crawlFileDir);
		if (!file.exists()) {
			file.mkdirs();
		}
		String crawlFilePath = crawlFileDir + fileName;
		// 若文件已存在，则删除
		file = new File(crawlFilePath);
		if (file.exists()) {
			file.delete();
		}
		try {
			logger.info("write data to file: " + crawlFilePath);
			FileUtils.writeWithNIO(data, FileUtils.DEFAULT_CHARSET_NAME,
					crawlFilePath);
		} catch (IOException e) {
			throw new MyTvException(
					"error occur while write crawled data to disk. filepaht ["
							+ crawlFilePath + "].", e);
		}
	}

	/**
	 * 获取运行路径
	 * 
	 * @return
	 */
	public static String getRunningPath(Class<?> clazz) {
		URL url = clazz.getProtectionDomain().getCodeSource().getLocation();
		String filePath = null;
		try {
			filePath = URLDecoder.decode(url.getPath(), "utf-8");// 转化为utf-8编码
		} catch (Exception e) {
			throw new MyTvException("invalid file path: " + url.getPath(), e);
		}
		if (filePath.endsWith(".jar")) {// 可执行jar包运行的结果里包含".jar"
			// 截取路径中的jar包名
			filePath = filePath.substring(0, filePath.lastIndexOf("/") + 1);
		}

		File file = new File(filePath);

		// /If this abstract pathname is already absolute, then the pathname
		// string is simply returned as if by the getPath method. If this
		// abstract pathname is the empty abstract pathname then the pathname
		// string of the current user directory, which is named by the system
		// property user.dir, is returned.
		filePath = file.getAbsolutePath();// 得到windows下的正确路径
		if (!filePath.endsWith(File.separator)) {
			filePath += File.separator;
		}

		return filePath;
	}

	public static boolean checkInvalidString(String string) {
		if (string.indexOf("'") != -1) {
			return false;
		}
		return true;
	}
	
}
 public class WebCrawler {
	private final static Logger logger = LoggerFactory
			.getLogger(WebCrawler.class);

	private final static BrowserVersion[] USER_AGENTS = new BrowserVersion[] {
			BrowserVersion.CHROME, BrowserVersion.FIREFOX_24,
			BrowserVersion.INTERNET_EXPLORER_11 };

	/**
	 * 使用htmlunit抓取网页
	 * 
	 * @param url
	 * @return
	 */
	public static Page crawl(String url) {
		return crawl(url, randomUserAgent());
	}

	/**
	 * 使用htmlunit抓取网页
	 * 
	 * @param url
	 * @param userAgent
	 * @param cookie
	 * @return
	 */
	public static Page crawl(String url, String userAgent) {
		return crawl(url, new BrowserVersion(WebCrawler.class.getName(), "1.0",
				userAgent, 1.0f));
	}

	/**
	 * 使用htmlunit抓取网页
	 * 
	 * @param url
	 * @param browserVersion
	 * @return
	 */
	private static Page crawl(String url, BrowserVersion browserVersion) {
		Proxy proxy = MyTvProxyManager.getInstance().pickProxy();
		WebClient webClient = new WebClient(browserVersion);
		if (proxy != null) {
			ProxyConfig pc = new ProxyConfig(proxy.getIp(), proxy.getPort());
			webClient.getOptions().setProxyConfig(pc);
		}
		webClient.getOptions().setJavaScriptEnabled(true);
		webClient.getOptions().setCssEnabled(false);
		webClient.getOptions().setThrowExceptionOnScriptError(false);
		webClient.setAjaxController(new NicelyResynchronizingAjaxController());

		try {
			logger.info("begin to get page: " + url
					+ (proxy != null ? ", using: " + proxy : ""));
			return webClient.getPage(url);
		} catch (FailingHttpStatusCodeException e) {
			throw new MyTvException("can't connect to " + url, e);
		} catch (MalformedURLException e) {
			throw new MyTvException("invalid url " + url, e);
		} catch (IOException e) {
			throw new MyTvException("error occur while connect to " + url, e);
		} finally {
			webClient.closeAllWindows();
		}
	}

	/**
	 * 取得随机浏览器标识
	 * 
	 * @return
	 */
	private static BrowserVersion randomUserAgent() {
		Random random = new Random();
		int max = USER_AGENTS.length;
		return USER_AGENTS[random.nextInt(max)];
	}
}

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.laudandjolynn.mytv.crawler.AbstractCrawler;
import com.laudandjolynn.mytv.event.AllTvStationCrawlEndEvent;
import com.laudandjolynn.mytv.event.CrawlEventListener;
import com.laudandjolynn.mytv.event.ProgramTableCrawlEndEvent;
import com.laudandjolynn.mytv.event.ProgramTableFoundEvent;
import com.laudandjolynn.mytv.event.TvStationFoundEvent;
import com.laudandjolynn.mytv.model.ProgramTable;
import com.laudandjolynn.mytv.model.TvStation;
import com.laudandjolynn.mytv.utils.Constant;
import com.laudandjolynn.mytv.utils.DateUtils;
import com.laudandjolynn.mytv.utils.MyTvUtils;
import com.laudandjolynn.mytv.utils.WebCrawler;


		if (!queryDate.equals(DateUtils.today())) {
			DomElement element = htmlPage.getElementById("date");
			element.setAttribute("readonly", "false");
			element.setAttribute("value", queryDate);
			element.setNodeValue(queryDate);
			element.setTextContent(queryDate);
			List<?> list = htmlPage.getByXPath("//div[@id='search_1']/a");
			HtmlAnchor anchor = (HtmlAnchor) list.get(0);
			try {
				htmlPage = anchor.click();
			} catch (IOException e) {
				logger.error("error occur while search program table of "
						+ stationName + " at spec date: " + queryDate, e);
				return null;
			}
		}
		String html = htmlPage.asXml();
		MyTvUtils.outputCrawlData(queryDate, html, queryDate
				+ Constant.UNDERLINE + getCrawlerName() + Constant.UNDERLINE
				+ stationName);
		List<ProgramTable> ptList = parseProgramTable(html);
		return ptList;
	}

	/**
	 * 解析电视台列表
	 * 
	 * @param html
	 * @return
	 */
	private List<TvStation> parseTvStation(String html) {
		Document doc = Jsoup.parse(html);
		Elements classifyElements = doc.select("ul.weishi a[href]");
		Elements stationElements = doc.select("div.md_left_right");
		List<TvStation> resultList = new ArrayList<TvStation>();

		for (int i = 0, size = classifyElements == null ? 0 : classifyElements
				.size(); i < size; i++) {
			Element classifyElement = classifyElements.get(i);
			String classify = classifyElement.text().trim();
			if (CITY.equals(classify)) {
				continue;
			}
			Element stationElement = stationElements.get(i);
			Elements stationTextElements = stationElement
					.select("dl h3 a.channel");
			for (int j = 0, ssize = stationTextElements == null ? 0
					: stationTextElements.size(); j < ssize; j++) {
				TvStation tv = new TvStation();
				String stationName = stationTextElements.get(j).text().trim();
				tv.setName(stationName);
				tv.setCity(null);
				tv.setClassify(classify);
				tv.setSequence(SEQUENCE.incrementAndGet());
				for (CrawlEventListener listener : listeners) {
					listener.itemFound(new TvStationFoundEvent(this, tv));
				}
				resultList.add(tv);
			}
		}
		Elements cityElements = stationElements.select("dl#cityList dd");
		for (int i = 0, size = cityElements == null ? 0 : cityElements.size(); i < size; i++) {
			Element cityElement = cityElements.get(i).select("h3 a[href]")
					.get(0);
			Elements cityStationElements = cityElements.get(i).select(
					"div.lv3 p a.channel");
			for (int j = 0, ssize = cityStationElements == null ? 0
					: cityStationElements.size(); j < ssize; j++) {
				TvStation tv = new TvStation();
				String stationName = cityStationElements.get(j).text().trim();
				tv.setName(stationName);
				tv.setCity(cityElement.text().trim());
				tv.setClassify(CITY);
				tv.setSequence(SEQUENCE.incrementAndGet());
				for (CrawlEventListener listener : listeners) {
					listener.itemFound(new TvStationFoundEvent(this, tv));
				}
				resultList.add(tv);
			}
		}
		return resultList;
	}

	/**
	 * 解析电视节目表
	 * 
	 * @param html
	 * @return
	 */
	private List<ProgramTable> parseProgramTable(String html) {
		Document doc = Jsoup.parse(html);
		List<ProgramTable> resultList = new ArrayList<ProgramTable>();
		Elements channelElements = doc.select("#channelTitle");
		String stationName = channelElements.get(0).text().trim();
		Elements weekElements = doc.select("#week li[rel]");
		int week = 0;
		String date = null;
		for (int i = 0, size = weekElements == null ? 0 : weekElements.size(); i < size; i++) {
			Element element = weekElements.get(i);
			if (element.hasClass("cur")) {
				week = i + 1;
				date = element.attr("rel").trim();
				break;
			}
		}
		Elements programElemens = doc.select("#epg_list div.content_c dl dd")
				.select("a.p_name_a, a.p_name");
		for (int i = 0, size = programElemens == null ? 0 : programElemens
				.size(); i < size; i++) {
			Element programElement = programElemens.get(i);
			String programContent = programElement.text().trim();
			String[] pc = programContent.split("\\s+");
			ProgramTable pt = new ProgramTable();
			pt.setAirDate(date);
			pt.setAirTime(date + " " + pc[0] + ":00");
			pt.setProgram(pc[1]);
			pt.setStationName(stationName);
			pt.setWeek(week);
			for (CrawlEventListener listener : listeners) {

				listener.itemFound(new ProgramTableFoundEvent(this, pt));
