public class Configs {

  public  static final String CONFIG_LOCATION = "config/app.properties";
    
  private static  Properties config = new Properties();
  
  static {
    try {
      ResourceUtil.load(config, CONFIG_LOCATION);
    } catch (IOException e) {
      Throwables.propagate(e);
    }
  }
  
  public static  String getString(String key){
    return config.getProperty(key);
  }
  
  
  public static List<String> getStrList(String key){
    String value =  config.getProperty(key);
    return Splitter.on(",").splitToList(value);
  }

  public static  Integer getInt(String key){
      String value =  config.getProperty(key);
      return Integer.parseInt(value);
  }

  public static   Long   getLong(String key){
    String value =  config.getProperty(key);
    return Long.parseLong(value);
  }
  
  
} public class HttpClientUtils {
	
	private static final Logger Log = LoggerFactory.getLogger(HttpClientUtils.class);
	private static PoolingHttpClientConnectionManager poolingmgr;
  
	private static final String HTTPCLIENT_CONNECTTIMEOUT = "httpclient.connectTimeout";
	private static final String HTTPCLIENT_SOCKETTIMEOUT  = "httpclient.socketTimeout";
	private static final String HTTPCLIENT_AGENT          = "httpclient.agent";
	//private static final String HTTPCLIENT_CONNMANAGERTIMEOUT          = "httpclient.connManagerTimeout";
	private static final String HTTPCLIENT_MAXTOTAL          = "httpclient.maxTotal";
	private static final String HTTPCLIENT_MAXPERROUTE         = "httpclient.maxPerRoute";
	private static final String HTTPCLIENT_RETRYCOUNT         = "httpclient.retryCount";
  
	private static HttpClient   httpClient;
	
	static {
	    Integer connectTimeOut = Configs.getInt(HTTPCLIENT_CONNECTTIMEOUT);
	    Integer socketTimeOut =  Configs.getInt(HTTPCLIENT_SOCKETTIMEOUT);
	    //Integer connManagerTimeOut =  Configs.getInt(HTTPCLIENT_CONNMANAGERTIMEOUT);
	    String  agentStr      = Configs.getString(HTTPCLIENT_AGENT);	    
	    Integer maxTotal =  Configs.getInt(HTTPCLIENT_MAXTOTAL);
	    Integer maxPerRoute =  Configs.getInt(HTTPCLIENT_MAXPERROUTE);
	    Integer retryCount = Configs.getInt(HTTPCLIENT_RETRYCOUNT);
	    
	    RequestConfig requestConfig =
	        RequestConfig.custom().setConnectTimeout(connectTimeOut).setSocketTimeout(socketTimeOut).build();
	    
	    DefaultHttpRequestRetryHandler dhr = new DefaultHttpRequestRetryHandler(retryCount,true);
	    
	    poolingmgr = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
	            .register("http", PlainConnectionSocketFactory.getSocketFactory()).build());
	    poolingmgr.setDefaultMaxPerRoute(maxPerRoute);
	    poolingmgr.setMaxTotal(maxTotal);
	    
	    httpClient =
	            HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).setUserAgent(agentStr)
	            .setConnectionManager(poolingmgr)
	            .setConnectionReuseStrategy(new NoConnectionReuseStrategy()).setRetryHandler(dhr).build();
	    }
	
	  /**
	   * get method获取字符串响应
	   * 
	   * @param request
	   * @return
	   */
	  public static String getContent(HttpUriRequest request) {
		  ResponseHandler<String> responseHandler = new BasicResponseHandlerEx();
		  String content = null;
		  try {
		  	content = httpClient.execute(request, responseHandler);
		  } catch (IOException e) {
		  	Log.error(request.getRequestLine().toString(), e);
		  	request.abort();
		  } catch (Exception e) {
		  	Log.error("Unkown Exception" + e.getMessage(),e);
		  	Throwables.propagate(e);
		  }
		  return content;
	  }
	  
	  /**
	   * get method获取字符串响应
	   * 
	   * @param url
	   * @return
	   */
	  public static String getContent(String url){
	    HttpGet httpGet = new HttpGet(url);
	    return getContent(httpGet);
	  }
	  
	  /**
	   * post 获取 字符串响应
	   * 
	   * @param url
	   * @param params
	   * @return
	   */
	  public static String post(String url,Map<String, String> params){
	    List<NameValuePair>   postParameters = new ArrayList<NameValuePair>();
	    if (!params.isEmpty()) {
	      for (Map.Entry<String, String> entry : params.entrySet()) {
	        postParameters.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
	      }
	    }
	    HttpPost post = new HttpPost(url);
	    try {
	      post.setEntity(new UrlEncodedFormEntity(postParameters));
	    } catch (UnsupportedEncodingException e) {
	      Throwables.propagate(e);
	    }
	    return getContent(post);
	  }

} public class RexxarNovelUtils {
	
	private static Logger logger = LoggerFactory.getLogger(RexxarNovelUtils.class);

	/**
	 * 生成小说唯一id，16位的bigint，  随机生成的三位数+时间戳倒序
	 * @return
	 */
	public static Long generatorNovelId()
	{
		String threeRandomSting =  RandomStringUtils.randomNumeric(3);
		String timeStamp = String.valueOf(System.currentTimeMillis());
		String idString = StringUtils.reverse(timeStamp) + threeRandomSting;
		
		return NumberUtils.toLong(idString);
	}
	
	public static String generatorNovelContentFilename(String chaptername)
	{
		checkNotNull(chaptername);
		return Hashing.md5().hashString(chaptername + "phoenix",Charset.forName("utf8")).toString();
	}
	
	public static void main(String[] args)
	{
		for (int i = 0; i < 10; i++) {
			logger.debug("{}",generatorNovelId());
			logger.debug("{}",generatorNovelContentFilename("第二章 大杀四方"));
		}		
	}
} public class RexxarPageUtils {

    /**
     * 解析html body获取charset，采用httpclient提供的getContentType等方法获取不到
     * @param html
     * @param charset
     * @return 默认返回 UTF-8
     */
    public static String getCharSetByBody(String html){    
    	String charset = "UTF-8";
    	Document document = Jsoup.parse(html);
        Elements elements = document.select("meta");    
        for(Element metaElement : elements){    
            if(metaElement!=null && StringUtils.isNotBlank(metaElement.attr("http-equiv")) && metaElement.attr("http-equiv").toLowerCase().equals("content-type")){    
                String content = metaElement.attr("content");    
                charset = getCharSet(content);    
                break;    
            }
        }
        return charset;    
    }
        
    /**  
     * 正则获取字符编码  
     * @param content  
     * @return  
     */    
    private static String getCharSet(String content){    
        String regex = ".*charset=([^;]*).*";    
        Pattern pattern = Pattern.compile(regex);    
        Matcher matcher = pattern.matcher(content);    
        if(matcher.find())    
            return matcher.group(1);    
        else    
            return null;    
    }  
    
    /**
     * 字符串编码转换的实现方法
     * @param str  待转换编码的字符串
     * @param newCharset 目标编码
     * @return
     * @throws UnsupportedEncodingException
     */
    public static String changeCharset(String str, String newCharset) throws UnsupportedEncodingException {
	     if (str != null) {
		      //用默认字符编码解码字符串。
		      byte[] bs = str.getBytes();
		      //用新的字符编码生成字符串
		      return new String(bs, newCharset);
	     }
	     return null;
    }
    
    /**
     * 字符串编码转换的实现方法
     * @param str  待转换编码的字符串
     * @param oldCharset 原编码
     * @param newCharset 目标编码
     * @return
     * @throws UnsupportedEncodingException
     */
    public static String changeCharset(String str, String oldCharset, String newCharset) throws UnsupportedEncodingException {
     if (str != null) {
	      //用旧的字符编码解码字符串。解码可能会出现异常。
	      byte[] bs = str.getBytes(oldCharset);
	      //用新的字符编码生成字符串
	      return new String(bs, newCharset);
     }
     
     return null;
    }
} @Component
public class BqgNovelExtract implements NovelExtract {

	private static Logger logger = LoggerFactory.getLogger(BqgNovelExtract.class);
	
	private static final String UPDATE_BETWEEN_NOW = "crawler.updateTimeBetweenNow";
	
	public static Integer updateBetweenNow = Configs.getInt(UPDATE_BETWEEN_NOW);
	
	public String getName(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:novel:book_name]");
		for (Element bookname : novelName) {
			logger.debug("bookname: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public String getAuthor(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:novel:author]");
		for (Element bookname : novelName) {
			logger.debug("author: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public String getImgUrl(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:image]");
		for (Element bookname : novelName) {
			logger.debug("imgurl: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public String getDescribe(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:description]");
		for (Element bookname : novelName) {
			logger.debug("describe: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public String getCagetory(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:novel:category]");
		for (Element bookname : novelName) {
			logger.debug("category: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public String getLatestChapterName(Document novelHtml) throws UnsupportedEncodingException {
		Elements novelName = novelHtml.select("meta[property=og:novel:latest_chapter_name]");
		for (Element bookname : novelName) {
			logger.debug("latest chapter name: " + bookname.attr("content"));
			return RexxarPageUtils.changeCharset(bookname.attr("content"), "utf8") ;
		}
		return null;
	}

	public String getStatus(Document novelHtml) {
		Elements novelName = novelHtml.select("meta[property=og:novel:status]");
		for (Element bookname : novelName) {
			logger.debug("status: " + bookname.attr("content"));
			return bookname.attr("content");
		}
		return null;
	}

	public short getDirectoryCount(Document novelHtml) {
		Elements novelDirectory = novelHtml.select("#list dd a");
		return (short) novelDirectory.size();
	}

	public String[] getALLDirectory(Document novelHtml) {
		Elements novelDirectory = novelHtml.select("#list dd a");
		
		Integer dirCount = novelDirectory.size();
		String[] dirs = new String[dirCount];
		
		for (int i = 0; i < dirCount; i++) {
			dirs[i] = novelDirectory.get(i).text();
		}
		
		return dirs;
	}

	public Pair<String[], String[]> getDirsAndUrls(Document novelHtml,String prexUrl) {
		Elements novelDirectory = novelHtml.select("#list dd a");
		
		Integer dirCount = novelDirectory.size();
		String[] dirs = new String[dirCount];
		String[] urls = new String[dirCount];
		
		for (int i = 0; i < dirCount; i++) {
			dirs[i] = novelDirectory.get(i).text();
			urls[i] = prexUrl + "/" + novelDirectory.get(i).attr("href");
		}
		
		return Pair.of(dirs, urls);
	}

	public String getNovelChapterContent(Document novelContentHtml) {
		Elements chapterContents = novelContentHtml.select("#content");
		for (Element chapterContent : chapterContents) {
			logger.debug("chapter content: " + chapterContent.text());
			try {
				return RexxarPageUtils.changeCharset(chapterContent.text(), "utf8") ;
			} catch (UnsupportedEncodingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		return null;
	}
	
	public String getLatestNovelUpdateTime(Document novelHtml) {
		Elements updateTimes = novelHtml.select("meta[property=og:novel:update_time]");
		for (Element updateTime : updateTimes) {
			logger.debug("updateTime: " + updateTime.attr("content"));
			return updateTime.attr("content");
		}
		return null;
	}	
	
	public boolean isNovelEnd(Document novelHtml) throws ParseException{
		String latestNovelUpdateTime = getLatestNovelUpdateTime(novelHtml);
		if(latestNovelUpdateTime == null){
			logger.warn("latestNovelUpdateTime is null,not update. novelhtml[{}]",novelHtml.toString());
			return true;
		}
		
		if (StringUtils.equals(getStatus(novelHtml), "完结") || 
				isUpdateTimeExpired(latestNovelUpdateTime)) {
			return true;
		}else{
			return false;
		}
	}
	
	private boolean isUpdateTimeExpired(String latestNovelUpdateTime) throws ParseException{		
		String currentTimeString = DateFormatUtils.format(new Date(), "yyyy-MM-dd");
		
		Date latestNovelDate = DateUtils.parseDate(latestNovelUpdateTime, "yyyy-MM-dd HH:mm");
		String latestNovelDateString = DateFormatUtils.format(latestNovelDate, "yyyy-MM-dd");
		logger.debug("latestNovelDateString: {}", latestNovelDateString);
		
		if(RexxarDateUtils.subTwoDays(currentTimeString, latestNovelDateString) >  updateBetweenNow){
			return true;
		}else {
			return false;
		}
	}
	
	public static void main(String[] args) {
		BqgNovelExtract bqgNovelExtract = new BqgNovelExtract();
		try {
			bqgNovelExtract.isUpdateTimeExpired("2015-11-06 17:02");
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

} public interface CrawlerTaskMapper {
    int countByExample(CrawlerTaskExample example);

    int deleteByExample(CrawlerTaskExample example);

    int deleteByPrimaryKey(Long id);

    int insert(CrawlerTask record);

    int insertSelective(CrawlerTask record);

    List<CrawlerTask> selectByExample(CrawlerTaskExample example);

    CrawlerTask selectByPrimaryKey(Long id);
    
    List<Long> selectIDs(CrawlerTaskExample example);

    int updateByExampleSelective(@Param("record") CrawlerTask record, @Param("example") CrawlerTaskExample example);

    int updateByExample(@Param("record") CrawlerTask record, @Param("example") CrawlerTaskExample example);

    int updateByPrimaryKeySelective(CrawlerTask record);

    int updateByPrimaryKey(CrawlerTask record);
} public interface NovelDirectoryMapper {
    int countByExample(NovelDirectoryExample example);

    int deleteByExample(NovelDirectoryExample example);

    int deleteByPrimaryKey(Long id);

    int insert(NovelDirectoryWithBLOBs record);

    int insertSelective(NovelDirectoryWithBLOBs record);

    List<NovelDirectoryWithBLOBs> selectByExampleWithBLOBs(NovelDirectoryExample example);

    List<NovelDirectory> selectByExample(NovelDirectoryExample example);

    NovelDirectoryWithBLOBs selectByPrimaryKey(Long id);

    int updateByExampleSelective(@Param("record") NovelDirectoryWithBLOBs record, @Param("example") NovelDirectoryExample example);

    int updateByExampleWithBLOBs(@Param("record") NovelDirectoryWithBLOBs record, @Param("example") NovelDirectoryExample example);

    int updateByExample(@Param("record") NovelDirectory record, @Param("example") NovelDirectoryExample example);

    int updateByPrimaryKeySelective(NovelDirectoryWithBLOBs record);

    int updateByPrimaryKeyWithBLOBs(NovelDirectoryWithBLOBs record);

    int updateByPrimaryKey(NovelDirectory record);
} public interface NovelOverviewMapper {
    int countByExample(NovelOverviewExample example);

    int deleteByExample(NovelOverviewExample example);

    int deleteByPrimaryKey(Long id);

    int insert(NovelOverview record);

    int insertSelective(NovelOverview record);

    List<NovelOverview> selectByExample(NovelOverviewExample example);

    NovelOverview selectByPrimaryKey(Long id);

    int updateByExampleSelective(@Param("record") NovelOverview record, @Param("example") NovelOverviewExample example);

    int updateByExample(@Param("record") NovelOverview record, @Param("example") NovelOverviewExample example);

    int updateByPrimaryKeySelective(NovelOverview record);

    int updateByPrimaryKey(NovelOverview record);
} public class CrawlerTask {
    private Long id;

    private String bookname;

    private String source;

    private String sourceUrl;

    private String backSource;

    private Short latestChapter;

    private String latestChapterName;

    private Short crawledChapter;

    private String crawledChapterName;

    private BigDecimal crawledPercent;

    private Boolean crawledStatus;

    private Date updateTime;

    private Date createTime;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getBookname() {
        return bookname;
    }

    public void setBookname(String bookname) {
        this.bookname = bookname == null ? null : bookname.trim();
    }

    public String getSource() {
        return source;
    }

    public void setSource(String source) {
        this.source = source == null ? null : source.trim();
    }

    public String getSourceUrl() {
        return sourceUrl;
    }

    public void setSourceUrl(String sourceUrl) {
        this.sourceUrl = sourceUrl == null ? null : sourceUrl.trim();
    }

    public String getBackSource() {
        return backSource;
    }

    public void setBackSource(String backSource) {
        this.backSource = backSource == null ? null : backSource.trim();
    }

    public Short getLatestChapter() {
        return latestChapter;
    }

    public void setLatestChapter(Short latestChapter) {
        this.latestChapter = latestChapter;
    }

    public String getLatestChapterName() {
        return latestChapterName;
    }

    public void setLatestChapterName(String latestChapterName) {
        this.latestChapterName = latestChapterName == null ? null : latestChapterName.trim();
    }

    public Short getCrawledChapter() {
        return crawledChapter;
    }

    public void setCrawledChapter(Short crawledChapter) {
        this.crawledChapter = crawledChapter;
    }

    public String getCrawledChapterName() {
        return crawledChapterName;
    }

    public void setCrawledChapterName(String crawledChapterName) {
        this.crawledChapterName = crawledChapterName == null ? null : crawledChapterName.trim();
    }

    public BigDecimal getCrawledPercent() {
        return crawledPercent;
    }

    public void setCrawledPercent(BigDecimal crawledPercent) {
        this.crawledPercent = crawledPercent;
    }

    public Boolean getCrawledStatus() {
        return crawledStatus;
    }

    public void setCrawledStatus(Boolean crawledStatus) {
        this.crawledStatus = crawledStatus;
    }

    public Date getUpdateTime() {
        return updateTime;
    }

    public void setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
    }

    public Date getCreateTime() {
        return createTime;
    }

    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
} public class CrawlerTaskExample {
    protected String orderByClause;

    protected boolean distinct;

    protected List<Criteria> oredCriteria;

    public CrawlerTaskExample() {
        oredCriteria = new ArrayList<Criteria>();
    }

    public void setOrderByClause(String orderByClause) {
        this.orderByClause = orderByClause;
    }

    public String getOrderByClause() {
        return orderByClause;
    }

    public void setDistinct(boolean distinct) {
        this.distinct = distinct;
    }

    public boolean isDistinct() {
        return distinct;
    }

    public List<Criteria> getOredCriteria() {
        return oredCriteria;
    }

    public void or(Criteria criteria) {
        oredCriteria.add(criteria);
    }

    public Criteria or() {
        Criteria criteria = createCriteriaInternal();
        oredCriteria.add(criteria);
        return criteria;
    }

    public Criteria createCriteria() {
        Criteria criteria = createCriteriaInternal();
        if (oredCriteria.size() == 0) {
            oredCriteria.add(criteria);
        }
        return criteria;
    }

    protected Criteria createCriteriaInternal() {
        Criteria criteria = new Criteria();
        return criteria;
    }

    public void clear() {
        oredCriteria.clear();
        orderByClause = null;
        distinct = false;
    }

    protected abstract static class GeneratedCriteria {
        protected List<Criterion> criteria;

        protected GeneratedCriteria() {
            super();
            criteria = new ArrayList<Criterion>();
        }

        public boolean isValid() {
            return criteria.size() > 0;
        }

        public List<Criterion> getAllCriteria() {
            return criteria;
        }

        public List<Criterion> getCriteria() {
            return criteria;
        }

        protected void addCriterion(String condition) {
            if (condition == null) {
                throw new RuntimeException("Value for condition cannot be null");
            }
            criteria.add(new Criterion(condition));
        }

        protected void addCriterion(String condition, Object value, String property) {
            if (value == null) {
                throw new RuntimeException("Value for " + property + " cannot be null");
            }
            criteria.add(new Criterion(condition, value));
        }

        protected void addCriterion(String condition, Object value1, Object value2, String property) {
            if (value1 == null || value2 == null) {
                throw new RuntimeException("Between values for " + property + " cannot be null");
            }
            criteria.add(new Criterion(condition, value1, value2));
        }

        public Criteria andIdIsNull() {
            addCriterion("id is null");
            return (Criteria) this;
        }

        public Criteria andIdIsNotNull() {
            addCriterion("id is not null");
            return (Criteria) this;
        }

        public Criteria andIdEqualTo(Long value) {
            addCriterion("id =", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdNotEqualTo(Long value) {
            addCriterion("id <>", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdGreaterThan(Long value) {
            addCriterion("id >", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdGreaterThanOrEqualTo(Long value) {
            addCriterion("id >=", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdLessThan(Long value) {
            addCriterion("id <", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdLessThanOrEqualTo(Long value) {
            addCriterion("id <=", value, "id");
            return (Criteria) this;
        }

        public Criteria andIdIn(List<Long> values) {
            addCriterion("id in", values, "id");
            return (Criteria) this;
        }

        public Criteria andIdNotIn(List<Long> values) {
            addCriterion("id not in", values, "id");
            return (Criteria) this;
        }

        public Criteria andIdBetween(Long value1, Long value2) {
            addCriterion("id between", value1, value2, "id");
            return (Criteria) this;
        }

        public Criteria andIdNotBetween(Long value1, Long value2) {
            addCriterion("id not between", value1, value2, "id");
            return (Criteria) this;
        }

        public Criteria andBooknameIsNull() {
            addCriterion("bookname is null");
            return (Criteria) this;
        }

        public Criteria andBooknameIsNotNull() {
            addCriterion("bookname is not null");
            return (Criteria) this;
        }

        public Criteria andBooknameEqualTo(String value) {
            addCriterion("bookname =", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameNotEqualTo(String value) {
            addCriterion("bookname <>", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameGreaterThan(String value) {
            addCriterion("bookname >", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameGreaterThanOrEqualTo(String value) {
            addCriterion("bookname >=", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameLessThan(String value) {
            addCriterion("bookname <", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameLessThanOrEqualTo(String value) {
            addCriterion("bookname <=", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameLike(String value) {
            addCriterion("bookname like", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameNotLike(String value) {
            addCriterion("bookname not like", value, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameIn(List<String> values) {
            addCriterion("bookname in", values, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameNotIn(List<String> values) {
            addCriterion("bookname not in", values, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameBetween(String value1, String value2) {
            addCriterion("bookname between", value1, value2, "bookname");
            return (Criteria) this;
        }

        public Criteria andBooknameNotBetween(String value1, String value2) {
            addCriterion("bookname not between", value1, value2, "bookname");
            return (Criteria) this;
        }

        public Criteria andSourceIsNull() {
            addCriterion("source is null");
            return (Criteria) this;
        }

        public Criteria andSourceIsNotNull() {
            addCriterion("source is not null");
            return (Criteria) this;
        }

        public Criteria andSourceEqualTo(String value) {
            addCriterion("source =", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceNotEqualTo(String value) {
            addCriterion("source <>", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceGreaterThan(String value) {
            addCriterion("source >", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceGreaterThanOrEqualTo(String value) {
            addCriterion("source >=", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceLessThan(String value) {
            addCriterion("source <", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceLessThanOrEqualTo(String value) {
            addCriterion("source <=", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceLike(String value) {
            addCriterion("source like", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceNotLike(String value) {
            addCriterion("source not like", value, "source");
            return (Criteria) this;
        }

        public Criteria andSourceIn(List<String> values) {
            addCriterion("source in", values, "source");
            return (Criteria) this;
        }

        public Criteria andSourceNotIn(List<String> values) {
            addCriterion("source not in", values, "source");
            return (Criteria) this;
        }

        public Criteria andSourceBetween(String value1, String value2) {
            addCriterion("source between", value1, value2, "source");
            return (Criteria) this;
        }

        public Criteria andSourceNotBetween(String value1, String value2) {
            addCriterion("source not between", value1, value2, "source");
            return (Criteria) this;
        }

        public Criteria andSourceUrlIsNull() {
            addCriterion("source_url is null");
            return (Criteria) this;
        }

        public Criteria andSourceUrlIsNotNull() {
            addCriterion("source_url is not null");
            return (Criteria) this;
        }

        public Criteria andSourceUrlEqualTo(String value) {
            addCriterion("source_url =", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlNotEqualTo(String value) {
            addCriterion("source_url <>", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlGreaterThan(String value) {
            addCriterion("source_url >", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlGreaterThanOrEqualTo(String value) {
            addCriterion("source_url >=", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlLessThan(String value) {
            addCriterion("source_url <", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlLessThanOrEqualTo(String value) {
            addCriterion("source_url <=", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlLike(String value) {
            addCriterion("source_url like", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlNotLike(String value) {
            addCriterion("source_url not like", value, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlIn(List<String> values) {
            addCriterion("source_url in", values, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlNotIn(List<String> values) {
            addCriterion("source_url not in", values, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlBetween(String value1, String value2) {
            addCriterion("source_url between", value1, value2, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andSourceUrlNotBetween(String value1, String value2) {
            addCriterion("source_url not between", value1, value2, "sourceUrl");
            return (Criteria) this;
        }

        public Criteria andBackSourceIsNull() {
            addCriterion("back_source is null");
            return (Criteria) this;
        }

        public Criteria andBackSourceIsNotNull() {
            addCriterion("back_source is not null");
            return (Criteria) this;
        }

        public Criteria andBackSourceEqualTo(String value) {
            addCriterion("back_source =", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceNotEqualTo(String value) {
            addCriterion("back_source <>", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceGreaterThan(String value) {
            addCriterion("back_source >", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceGreaterThanOrEqualTo(String value) {
            addCriterion("back_source >=", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceLessThan(String value) {
            addCriterion("back_source <", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceLessThanOrEqualTo(String value) {
            addCriterion("back_source <=", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceLike(String value) {
            addCriterion("back_source like", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceNotLike(String value) {
            addCriterion("back_source not like", value, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceIn(List<String> values) {
            addCriterion("back_source in", values, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceNotIn(List<String> values) {
            addCriterion("back_source not in", values, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceBetween(String value1, String value2) {
            addCriterion("back_source between", value1, value2, "backSource");
            return (Criteria) this;
        }

        public Criteria andBackSourceNotBetween(String value1, String value2) {
            addCriterion("back_source not between", value1, value2, "backSource");
            return (Criteria) this;
        }

        public Criteria andLatestChapterIsNull() {
            addCriterion("latest_chapter is null");
            return (Criteria) this;
        }

        public Criteria andLatestChapterIsNotNull() {
            addCriterion("latest_chapter is not null");
            return (Criteria) this;
        }

        public Criteria andLatestChapterEqualTo(Short value) {
            addCriterion("latest_chapter =", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNotEqualTo(Short value) {
            addCriterion("latest_chapter <>", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterGreaterThan(Short value) {
            addCriterion("latest_chapter >", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterGreaterThanOrEqualTo(Short value) {
            addCriterion("latest_chapter >=", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterLessThan(Short value) {
            addCriterion("latest_chapter <", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterLessThanOrEqualTo(Short value) {
            addCriterion("latest_chapter <=", value, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterIn(List<Short> values) {
            addCriterion("latest_chapter in", values, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNotIn(List<Short> values) {
            addCriterion("latest_chapter not in", values, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterBetween(Short value1, Short value2) {
            addCriterion("latest_chapter between", value1, value2, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNotBetween(Short value1, Short value2) {
            addCriterion("latest_chapter not between", value1, value2, "latestChapter");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameIsNull() {
            addCriterion("latest_chapter_name is null");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameIsNotNull() {
            addCriterion("latest_chapter_name is not null");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameEqualTo(String value) {
            addCriterion("latest_chapter_name =", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameNotEqualTo(String value) {
            addCriterion("latest_chapter_name <>", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameGreaterThan(String value) {
            addCriterion("latest_chapter_name >", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameGreaterThanOrEqualTo(String value) {
            addCriterion("latest_chapter_name >=", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameLessThan(String value) {
            addCriterion("latest_chapter_name <", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameLessThanOrEqualTo(String value) {
            addCriterion("latest_chapter_name <=", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameLike(String value) {
            addCriterion("latest_chapter_name like", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameNotLike(String value) {
            addCriterion("latest_chapter_name not like", value, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameIn(List<String> values) {
            addCriterion("latest_chapter_name in", values, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameNotIn(List<String> values) {
            addCriterion("latest_chapter_name not in", values, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameBetween(String value1, String value2) {
            addCriterion("latest_chapter_name between", value1, value2, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andLatestChapterNameNotBetween(String value1, String value2) {
            addCriterion("latest_chapter_name not between", value1, value2, "latestChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterIsNull() {
            addCriterion("crawled_chapter is null");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterIsNotNull() {
            addCriterion("crawled_chapter is not null");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterEqualTo(Short value) {
            addCriterion("crawled_chapter =", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNotEqualTo(Short value) {
            addCriterion("crawled_chapter <>", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterGreaterThan(Short value) {
            addCriterion("crawled_chapter >", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterGreaterThanOrEqualTo(Short value) {
            addCriterion("crawled_chapter >=", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterLessThan(Short value) {
            addCriterion("crawled_chapter <", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterLessThanOrEqualTo(Short value) {
            addCriterion("crawled_chapter <=", value, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterIn(List<Short> values) {
            addCriterion("crawled_chapter in", values, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNotIn(List<Short> values) {
            addCriterion("crawled_chapter not in", values, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterBetween(Short value1, Short value2) {
            addCriterion("crawled_chapter between", value1, value2, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNotBetween(Short value1, Short value2) {
            addCriterion("crawled_chapter not between", value1, value2, "crawledChapter");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameIsNull() {
            addCriterion("crawled_chapter_name is null");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameIsNotNull() {
            addCriterion("crawled_chapter_name is not null");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameEqualTo(String value) {
            addCriterion("crawled_chapter_name =", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameNotEqualTo(String value) {
            addCriterion("crawled_chapter_name <>", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameGreaterThan(String value) {
            addCriterion("crawled_chapter_name >", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameGreaterThanOrEqualTo(String value) {
            addCriterion("crawled_chapter_name >=", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameLessThan(String value) {
            addCriterion("crawled_chapter_name <", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameLessThanOrEqualTo(String value) {
            addCriterion("crawled_chapter_name <=", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameLike(String value) {
            addCriterion("crawled_chapter_name like", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameNotLike(String value) {
            addCriterion("crawled_chapter_name not like", value, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameIn(List<String> values) {
            addCriterion("crawled_chapter_name in", values, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameNotIn(List<String> values) {
            addCriterion("crawled_chapter_name not in", values, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameBetween(String value1, String value2) {
            addCriterion("crawled_chapter_name between", value1, value2, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledChapterNameNotBetween(String value1, String value2) {
            addCriterion("crawled_chapter_name not between", value1, value2, "crawledChapterName");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentIsNull() {
            addCriterion("crawled_percent is null");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentIsNotNull() {
            addCriterion("crawled_percent is not null");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentEqualTo(BigDecimal value) {
            addCriterion("crawled_percent =", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentNotEqualTo(BigDecimal value) {
            addCriterion("crawled_percent <>", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentGreaterThan(BigDecimal value) {
            addCriterion("crawled_percent >", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentGreaterThanOrEqualTo(BigDecimal value) {
            addCriterion("crawled_percent >=", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentLessThan(BigDecimal value) {
            addCriterion("crawled_percent <", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentLessThanOrEqualTo(BigDecimal value) {
            addCriterion("crawled_percent <=", value, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentIn(List<BigDecimal> values) {
            addCriterion("crawled_percent in", values, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentNotIn(List<BigDecimal> values) {
            addCriterion("crawled_percent not in", values, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentBetween(BigDecimal value1, BigDecimal value2) {
            addCriterion("crawled_percent between", value1, value2, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledPercentNotBetween(BigDecimal value1, BigDecimal value2) {
            addCriterion("crawled_percent not between", value1, value2, "crawledPercent");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusIsNull() {
            addCriterion("crawled_status is null");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusIsNotNull() {
            addCriterion("crawled_status is not null");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusEqualTo(Boolean value) {
            addCriterion("crawled_status =", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusNotEqualTo(Boolean value) {
            addCriterion("crawled_status <>", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusGreaterThan(Boolean value) {
            addCriterion("crawled_status >", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusGreaterThanOrEqualTo(Boolean value) {
            addCriterion("crawled_status >=", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusLessThan(Boolean value) {
            addCriterion("crawled_status <", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusLessThanOrEqualTo(Boolean value) {
            addCriterion("crawled_status <=", value, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusIn(List<Boolean> values) {
            addCriterion("crawled_status in", values, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusNotIn(List<Boolean> values) {
            addCriterion("crawled_status not in", values, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusBetween(Boolean value1, Boolean value2) {
            addCriterion("crawled_status between", value1, value2, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andCrawledStatusNotBetween(Boolean value1, Boolean value2) {
            addCriterion("crawled_status not between", value1, value2, "crawledStatus");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeIsNull() {
            addCriterion("update_time is null");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeIsNotNull() {
            addCriterion("update_time is not null");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeEqualTo(Date value) {
            addCriterion("update_time =", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeNotEqualTo(Date value) {
            addCriterion("update_time <>", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeGreaterThan(Date value) {
            addCriterion("update_time >", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeGreaterThanOrEqualTo(Date value) {
            addCriterion("update_time >=", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeLessThan(Date value) {
            addCriterion("update_time <", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeLessThanOrEqualTo(Date value) {
            addCriterion("update_time <=", value, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeIn(List<Date> values) {
            addCriterion("update_time in", values, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeNotIn(List<Date> values) {
            addCriterion("update_time not in", values, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeBetween(Date value1, Date value2) {
            addCriterion("update_time between", value1, value2, "updateTime");
            return (Criteria) this;
        }

        public Criteria andUpdateTimeNotBetween(Date value1, Date value2) {
            addCriterion("update_time not between", value1, value2, "updateTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeIsNull() {
            addCriterion("create_time is null");
            return (Criteria) this;
        }

        public Criteria andCreateTimeIsNotNull() {
            addCriterion("create_time is not null");
            return (Criteria) this;
        }

        public Criteria andCreateTimeEqualTo(Date value) {
            addCriterion("create_time =", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeNotEqualTo(Date value) {
            addCriterion("create_time <>", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeGreaterThan(Date value) {
            addCriterion("create_time >", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeGreaterThanOrEqualTo(Date value) {
            addCriterion("create_time >=", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeLessThan(Date value) {
            addCriterion("create_time <", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeLessThanOrEqualTo(Date value) {
            addCriterion("create_time <=", value, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeIn(List<Date> values) {
            addCriterion("create_time in", values, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeNotIn(List<Date> values) {
            addCriterion("create_time not in", values, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeBetween(Date value1, Date value2) {
            addCriterion("create_time between", value1, value2, "createTime");
            return (Criteria) this;
        }

        public Criteria andCreateTimeNotBetween(Date value1, Date value2) {
            addCriterion("create_time not between", value1, value2, "createTime");
            return (Criteria) this;
        }
    }

    public static class Criteria extends GeneratedCriteria {

        protected Criteria() {
            super();
        }
    }

    public static class Criterion {
        private String condition;

        private Object value;

        private Object secondValue;

        private boolean noValue;

        private boolean singleValue;

        private boolean betweenValue;

        private boolean listValue;

        private String typeHandler;

        public String getCondition() {
            return condition;
        }

        public Object getValue() {
            return value;
        }

        public Object getSecondValue() {
            return secondValue;
        }

        public boolean isNoValue() {
            return noValue;
        }

        public boolean isSingleValue() {
            return singleValue;
        }

        public boolean isBetweenValue() {
            return betweenValue;
        }

        public boolean isListValue() {
            return listValue;
        }

        public String getTypeHandler() {
            return typeHandler;
        }

        protected Criterion(String condition) {
            super();
            this.condition = condition;
            this.typeHandler = null;
            this.noValue = true;
        }

        protected Criterion(String condition, Object value, String typeHandler) {
            super();
            this.condition = condition;
            this.value = value;
            this.typeHandler = typeHandler;
            if (value instanceof List<?>) {
                this.listValue = true;
            } else {
                this.singleValue = true;
            }
        }

        protected Criterion(String condition, Object value) {
            this(condition, value, null);
        }

        protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
            super();
            this.condition = condition;
            this.value = value;
            this.secondValue = secondValue;
            this.typeHandler = typeHandler;
            this.betweenValue = true;
        }

        protected Criterion(String condition, Object value, Object secondValue) {
            this(condition, value, secondValue, null);
        }
    }
} public static class Criteria extends GeneratedCriteria {

    protected Criteria() {
        super();
    }
} public class NovelDirectoryWithBLOBs extends NovelDirectory {
    private String text;

    private String latestText;

    public String getText() {
        return text;
    }

    public void setText(String text) {
        this.text = text == null ? null : text.trim();
    }

    public String getLatestText() {
        return latestText;
    }

    public void setLatestText(String latestText) {
        this.latestText = latestText == null ? null : latestText.trim();
    }
} public class NovelOverview {
    private Long id;

    private String bookname;

    private String author;

    private String imgUrl;

    private String novelDescribe;

    private String category;

    private Short crawledChapter;

    private String crawledChapterName;

    private String novelStatus;

    private BigDecimal novelScore;

    private String publishSource;

    private Boolean preferGender;

    private Date updateTime;

    private Date createTime;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getBookname() {
        return bookname;
    }

    public void setBookname(String bookname) {
        this.bookname = bookname == null ? null : bookname.trim();
    }

    public String getAuthor() {
        return author;
    }

    public void setAuthor(String author) {
        this.author = author == null ? null : author.trim();
    }

    public String getImgUrl() {
        return imgUrl;
    }

    public void setImgUrl(String imgUrl) {
        this.imgUrl = imgUrl == null ? null : imgUrl.trim();
    }

    public String getNovelDescribe() {
        return novelDescribe;
    }

    public void setNovelDescribe(String novelDescribe) {
        this.novelDescribe = novelDescribe == null ? null : novelDescribe.trim();
    }

    public String getCategory() {
        return category;
    }

    public void setCategory(String category) {
        this.category = category == null ? null : category.trim();
    }

    public Short getCrawledChapter() {
        return crawledChapter;
    }

    public void setCrawledChapter(Short crawledChapter) {
        this.crawledChapter = crawledChapter;
    }

    public String getCrawledChapterName() {
        return crawledChapterName;
    }

    public void setCrawledChapterName(String crawledChapterName) {
        this.crawledChapterName = crawledChapterName == null ? null : crawledChapterName.trim();
    }

    public String getNovelStatus() {
        return novelStatus;
    }

    public void setNovelStatus(String novelStatus) {
        this.novelStatus = novelStatus == null ? null : novelStatus.trim();
    }

    public BigDecimal getNovelScore() {
        return novelScore;
    }

    public void setNovelScore(BigDecimal novelScore) {
        this.novelScore = novelScore;
    }

    public String getPublishSource() {
        return publishSource;
    }

    public void setPublishSource(String publishSource) {
        this.publishSource = publishSource == null ? null : publishSource.trim();
    }

    public Boolean getPreferGender() {
        return preferGender;
    }

    public void setPreferGender(Boolean preferGender) {
        this.preferGender = preferGender;
    }

    public Date getUpdateTime() {
        return updateTime;
    }

    public void setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
    }

    public Date getCreateTime() {
        return createTime;
    }

    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
}
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import com.rexxar.common.global.Configs;
import com.rexxar.common.utils.HttpClientUtils;
import com.rexxar.common.utils.RexxarNovelUtils;
import com.rexxar.common.utils.RexxarPageUtils;
import com.rexxar.common.extract.BqgNovelExtract;
import com.rexxar.dao.CrawlerTaskMapper;
import com.rexxar.dao.NovelDirectoryMapper;
import com.rexxar.dao.NovelOverviewMapper;
import com.rexxar.dao.model.CrawlerTask;
import com.rexxar.dao.model.CrawlerTaskExample;
import com.rexxar.dao.model.CrawlerTaskExample.Criteria;
import com.rexxar.dao.model.NovelDirectoryWithBLOBs;
import com.rexxar.dao.model.NovelOverview;
package com.rexxar.crawler;







/**
 * 定时抓取最新发布的小说
 *
 */
public class NewNovelCrawler
{	
	private static Logger logger = LoggerFactory.getLogger(NewNovelCrawler.class);
	
  	private static final String CRAWLE_Filedir = "crawler.filedir";
	public static String crawleFiledir = Configs.getString(CRAWLE_Filedir);
	
	//public static String sourceUrl = "http://www.biquge.la/book/2/";
	
	@Autowired
	private CrawlerTaskMapper crawlerTaskMapper;
	
	@Autowired
	private NovelOverviewMapper novelOverviewMapper;
	
	@Autowired
	private NovelDirectoryMapper novelDirectoryMapper;
	
	@Autowired
	private BqgNovelExtract bqgNovelExtract;
	
	/**
	 * 抓取新的小说，初始化抓取任务表和小说概览表， 写入两个表用事务来管理
	 * @throws IOException 
	 */
	@Transactional()
	public void CrawleNewNovel(String sourceUrl) throws IOException
	{
		String sHtml = HttpClientUtils.getContent(sourceUrl);
		if (sHtml == null) {
			logger.error("fetch {} error, not continue crawl",sourceUrl);
			return;
		}
		
		String charset =  RexxarPageUtils.getCharSetByBody(sHtml);
		//logger.debug("Response charset:  " +  charset);
		
		//httpclient default charset:  ISO-8859-1
		String cleanHtml = RexxarPageUtils.changeCharset(sHtml, "ISO-8859-1", charset);
		
		Document novelHtml = Jsoup.parse(cleanHtml);
		
		//小说已完结不主动抓取
		try {
			if (bqgNovelExtract.isNovelEnd(novelHtml)) {
				logger.info("novel [{}] is end, do not take the initiative to crawl", bqgNovelExtract.getName(novelHtml));
				return;
			}
		} catch (ParseException e) {
			e.printStackTrace();
		}
				
		//小说在抓取任务中存在同名，不重复抓取
		CrawlerTaskExample cTaskExample = new CrawlerTaskExample();
		Criteria criteria = cTaskExample.createCriteria();
		criteria.andBooknameEqualTo(bqgNovelExtract.getName(novelHtml));
		ArrayList<CrawlerTask> crawlerTask = (ArrayList<CrawlerTask>) crawlerTaskMapper.selectByExample(cTaskExample);
		if (crawlerTask.size() > 0) {
			logger.info("novel [{}] already in rexxar task, do not repeat crawl", bqgNovelExtract.getName(novelHtml));
			return;
		}
		
		Long novelId = RexxarNovelUtils.generatorNovelId();
		//创建抓取任务
		initCrawlerTask(novelHtml,novelId,sourceUrl);
		//创建小说概览
		initNovelOverview(novelHtml, novelId);
		//创建小说目录存储
		initNovelDirectory(novelHtml,novelId);
		//创建小说章节内容存储， 直接文件存储  （如果web系统，可以考虑存储为html页面后直接访问）
		initDownloadContentDir(novelId);
	}
	
	private void initCrawlerTask(Document novelHtml,Long novelId,String sourceUrl) throws UnsupportedEncodingException
	{
		CrawlerTask ct = new CrawlerTask();
		
		String latestChapterName = bqgNovelExtract.getLatestChapterName(novelHtml);
		String bookname = bqgNovelExtract.getName(novelHtml);
		Short latestChapter = bqgNovelExtract.getDirectoryCount(novelHtml);
		Date date = new Date();
		Timestamp timeStamp = new Timestamp(date.getTime());
		
		ct.setId(novelId);
		ct.setBookname(bookname);
		ct.setSource("biquge");
		ct.setSourceUrl(sourceUrl);
		ct.setBackSource("");
		ct.setLatestChapter(latestChapter);
		ct.setLatestChapterName(latestChapterName);
		ct.setCrawledChapter((short) 0);
		ct.setCrawledChapterName("");  
		ct.setCrawledPercent(BigDecimal.valueOf(0.0000));     
		ct.setCrawledStatus(true);
		ct.setUpdateTime(timeStamp);
		ct.setCreateTime(timeStamp);
		
		crawlerTaskMapper.insert(ct);
		
		logger.info("novel [{}] initCrawlerTask success", bookname);
	}
	
	private void initNovelOverview(Document novelHtml,Long novelId)
	{
		NovelOverview novelOverview = new NovelOverview();