public interface Article
{
  /**
   * The original URL of the article.
   * 
   * @return The original URL of the article.
   */
  String getOriginalUrl();

  /**
   * Some sources of articles (such as Instapaper) provide a way to see any web
   * page formatted by their service (even if you haven't saved it in your
   * article list). This will be returned here if available.
   * 
   * @return A URL to view the article via the Source (such as Instapaper), or
   *         <code>null</code> if not available.
   */
  String getSourceUrl();

  /**
   * A plain text summary of the article, if available. May be an empty string
   * or <code>null</code> if none available.
   * 
   * @return The summary, and empty string or <code>null</code>.
   */
  String getSummary();

  /**
   * The title of the article.
   * 
   * @return The title of the article.
   */
  String getTitle();

  /**
   * <p>
   * The HTML text of article. This is not a full HTML document, just the
   * content.
   * </p>
   * 
   * <p>
   * Developers note: If you cache the text, and don't fetch it from the source
   * each time this method is called, then you should return a copy that it is
   * safe for the calling code to modify.
   * </p>
   * 
   * @return An {@link Element} containing the text of the article.
   * @throws IOException
   */
  Element text() throws IOException;
} public class Erudite
{
  private static class Footnote
  {
    private final String url;
    private final String num;
    private final String text;

    Footnote( final String url, final int num, final String text )
    {
      this(url, String.valueOf(num), text);
    }

    Footnote( final String url, final String num, final String text )
    {
      this.url = url;
      this.num = num;
      this.text = text;
    }

    @Override
    public boolean equals( final Object obj )
    {
      if ( this == obj )
        return true;
      if ( obj == null )
        return false;
      if ( getClass() != obj.getClass() )
        return false;
      Footnote other = (Footnote) obj;
      if ( num == null )
      {
        if ( other.num != null )
          return false;
      }
      else if ( !num.equals(other.num) )
        return false;
      if ( text == null )
      {
        if ( other.text != null )
          return false;
      }
      else if ( !text.equals(other.text) )
        return false;
      if ( url == null )
      {
        if ( other.url != null )
          return false;
      }
      else if ( !url.equals(other.url) )
        return false;
      return true;
    }

    @Override
    public int hashCode()
    {
      final int prime = 31;
      int result = 1;
      result = prime * result + ((num == null) ? 0 : num.hashCode());
      result = prime * result + ((text == null) ? 0 : text.hashCode());
      result = prime * result + ((url == null) ? 0 : url.hashCode());
      return result;
    }

    String getNum()
    {
      return num;
    }

    String getText()
    {
      return text;
    }

    String getUrl()
    {
      return url;
    }
  }

  private final Logger log = LoggerFactory.getLogger(Erudite.class);

  /**
   * Process an article and produce a complete HTML document. This object is
   * thread-safe.
   * 
   * @param article
   *        The article to process.
   * @param source
   *        The source of the article.
   * @param template
   *        The template used to produce the full HTML document.
   * @param image_handler
   *        Used for downloading any images we find.
   * @param do_footnotes
   *        Should we produce footnotes for links?
   * @param hacker_news_urls
   *        The hacker news discussion URLs for the article.
   * @return The full HTML document representing the article.
   * @throws IOException
   */
  public Document process( final Article article, final Source source,
      final Template template, final ImageHandler image_handler,
      final boolean do_footnotes, final ImmutableList<String> hacker_news_urls )
    throws IOException
  {
    log.debug("Processing {}", article.getTitle());
    final Document doc = template.getDocument();
    doc.title(article.getTitle());
    setTextElementByClass(doc, "erudite_title", article.getTitle());
    setAByClass(doc, "erudite_original_url", article.getOriginalUrl());
    setAByClass(doc, "erudite_original_link", article.getOriginalUrl(), null);
    insertPlug(doc, source);

    insertSourceMeta(doc, article, source);
    insertHackerNewsMeta(doc, hacker_news_urls);

    final Element content = article.text();
    if ( do_footnotes )
      insertFootnotes(doc, content, article.getOriginalUrl());
    else
      removeFootnotes(doc);

    handleImages(content, image_handler);

    insertContent(doc, content);
    return doc;
  }

  private void handleImages( final Element content,
      final ImageHandler image_handler ) throws IOException
  {
    final Map<String, String> mappings = Colls.newHashMap();
    final List<Element> yank = Colls.newArrayList();

    final Elements images = content.getElementsByTag("img");
    if ( images != null )
      for ( Element image : images )
      {
        final String orig_source = image.attr("src");
        if ( Str.isEmpty(orig_source) )
        {
          yank.add(image);
        }
        else if ( image_handler == null )
        {
          log.debug("No image handler, so removing image from the HTML: {}",
            orig_source);
          yank.add(image);
        }
        else
        {
          String new_source = mappings.get(orig_source);
          if ( new_source == null )
          {
            new_source = image_handler.image(orig_source);
            if ( new_source != null )
              mappings.put(orig_source, new_source);
          }
          if ( new_source != null )
          {
            log.debug("Processed image from {} => {}", orig_source, new_source);
            image.attr("src", new_source);
          }
          else
          {
            log.debug(
              "Could not retrieve image, so removing from the HTML: {}",
              orig_source);
            yank.add(image);
          }
        }
      }

    for ( Element image : yank )
      image.remove();
  }

  private void insertContent( final Document doc, final Element element )
  {
    final Element content = doc.getElementById("erudite_contents");
    content.empty();
    content.appendChild(element);
  }

  private void insertFootnotes( final Document doc, final Element content,
      final String src_url )
  {
    final Element footnotes_element = doc.getElementById("erudite_footnotes");
    if ( footnotes_element == null )
    {
      log.debug("No footnotes element in template!");
      removeFootnotes(doc);
      return;
    }

    int next_footnote_num = 1;
    final List<Footnote> footnotes_list = Colls.newArrayList();
    final Map<String, Footnote> footnotes_map = Colls.newHashMap();
    final Tag SUP_TAG = Tag.valueOf("sup");
    final Tag LINK_TAG = Tag.valueOf("a");

    final Elements links = content.getElementsByTag("a");
    if ( links != null )
      for ( Element link : links )
      {
        final String href = link.attr("href");
        if ( Str.isNotEmpty(href) //
            && (href.charAt(0) != '#') //
            && Str.notEquals(href, src_url) )
        {
          final String text = Str.trimToEmpty(link.text());
          if ( !isSameUrl(href, text) )
          {
            boolean first = false;
            Footnote footnote = footnotes_map.get(href);
            if ( footnote == null )
            {
              footnote = new Footnote(href, next_footnote_num++, text);
              footnotes_list.add(footnote);
              footnotes_map.put(href, footnote);
              first = true;
            }
            else if ( footnote.getText().isEmpty() && (!text.isEmpty()) )
            {
              // If the original footnote didn't have any text, replace with
              // one that did.
              final int pos = footnotes_list.indexOf(footnote);
              footnote = new Footnote(href, footnote.getNum(), text);
              footnotes_list.set(pos, footnote);
              footnotes_map.put(href, footnote);
            }

            // Create insertion.
            {
              final Element tag = new Element(LINK_TAG, Str.EMPTY);
              tag.addClass("erudite_footnote_source");
              tag.attr("href", "#erudite_footnote_" + footnote.getNum());
              tag.attr("title", "Goto footnote");
              if ( first )
                tag.attr("id", "erudite_sourcenote_" + footnote.getNum());
              {
                final Element sup = new Element(SUP_TAG, Str.EMPTY);
                sup.text("[" + footnote.getNum() + ']');
                tag.appendChild(sup);
              }
              link.after(tag);
              link.after("&nbsp;");
            }
          }
        }
      }

    if ( footnotes_list.isEmpty() )
    {
      log.debug("No footnotes for this article.");
      removeFootnotes(doc);
      return;
    }

    // Create list of footnotes.
    final Tag DT_TAG = Tag.valueOf("dt");
    final Tag DD_TAG = Tag.valueOf("dd");
    for ( Footnote footnote : footnotes_list )
    {
      final String text = Utils.abbreviate(footnote.getText(), 40);

      {
        final Element dt = new Element(DT_TAG, Str.EMPTY);
        dt.attr("id", "erudite_footnote_" + footnote.getNum());
        footnotes_element.appendChild(dt);

        {
          final Element src_link = new Element(LINK_TAG, Str.EMPTY);
          src_link.attr("href", "#erudite_sourcenote_" + footnote.getNum());
          if ( Str.isNotEmpty(text) )
            src_link.attr("title", text);
          src_link.text("[" + footnote.getNum() + ']');
          dt.appendChild(src_link);
        }

        dt.appendText(" ");
        dt.appendText(text);
      }

      {
        final Element dd = new Element(DD_TAG, Str.EMPTY);
        footnotes_element.appendChild(dd);
        final Element link = new Element(LINK_TAG, Str.EMPTY);
        link.attr("href", footnote.getUrl());
        link.attr("class", "erudite_url_url");
        if ( Str.isNotEmpty(text) )
          link.attr("title", text);
        link.text(footnote.getUrl());
        dd.appendChild(link);
      }
    }

    if ( footnotes_list.size() == 1 )
      log.debug("Inserted one footnote.");
    else
      log.debug("Inserted {} footnotes.", footnotes_list.size());
  }

  private void insertHackerNewsMeta( final Document doc,
      final ImmutableList<String> hacker_news_urls )
  {
    // #### No hacker news links.
    if ( (hacker_news_urls == null) || hacker_news_urls.isEmpty() )
    {
      removeByClass(doc, "erudite_hn_info");
      removeByClass(doc, "erudite_hn_infos");
      return;
    }

    // #### One hacker news link.
    if ( hacker_news_urls.size() == 1 )
    {
      removeByClass(doc, "erudite_hn_infos");
      final String url = hacker_news_urls.get(0);
      setAByClass(doc, "erudite_hn_url", url);
      setAByClass(doc, "erudite_hn_link", url, null);
      return;
    }

    // #### Several hacker news links.
    removeByClass(doc, "erudite_hn_info");

    final Elements links = doc.getElementsByClass("erudite_hn_links");
    for ( final Element el : links )
      el.empty();

    final Elements urls = doc.getElementsByClass("erudite_hn_urls");
    for ( final Element el : urls )
      el.empty();

    int link_num = 0;
    boolean first = true;
    for ( final String hn_url : hacker_news_urls )
    {
      for ( final Element el : links )
      {
        if ( !first )
          el.appendText(", ");
        el.appendElement("a").attr("class", "erudite_url_link erudite_hn_link")
            .attr("href", hn_url).text("#" + String.valueOf(++link_num));
      }

      for ( final Element el : urls )
      {
        if ( !first )
          el.appendText(", ");
        el.appendElement("a").attr("class", "erudite_url_url erudite_hn_url")
            .attr("href", hn_url).text(hn_url);
      }
      first = false;
    }
  }

  private int insertPlug( final Document doc, final Source source )
  {
    final Elements plugs = doc.getElementsByClass("erudite_source");
    if ( (plugs == null) || (plugs.size() <= 0) )
    {
      log.debug("Template missing source plug element.");
      return 0;
    }
    for ( final Element plug : plugs )
    {
      plug.empty();
      plug.html(source.getViaHtml());
    }
    return plugs.size();
  }

  private void insertSourceMeta( final Document doc, final Article article,
      final Source source )
  {
    final String source_url = article.getSourceUrl();
    if ( Str.isEmpty(source_url) )
    {
      removeByClass(doc, "erudite_source_info");
    }
    else
    {
      setAByClass(doc, "erudite_source_url", source_url);
      setAByClass(doc, "erudite_source_link", source_url, null);
      setTextElementByClass(doc, "erudite_source_name", source.getName());
    }
  }

  private boolean isSameUrl( final String href, final String text )
  {
    if ( href.equals(text) )
      return true;
    if ( !text.startsWith("http://") )
      if ( href.equals("http://" + text) )
        return true;
    return false;
  }

  private int removeByClass( final Document doc, final String clsname )
  {
    final Elements elements = doc.getElementsByClass(clsname);
    if ( elements == null )
      return 0;
    final int sz = elements.size();
    for ( final Element element : ImmutableList.copyOf(elements) )
    {
      element.empty();
      element.remove();
    }
    return sz;
  }

  private boolean removeById( final Document doc, final String id )
  {
    Element element = doc.getElementById(id);
    if ( element == null )
      return false;
    element.empty();
    element.remove();
    return true;
  }

  private void removeFootnotes( final Document doc )
  {
    if ( !removeById(doc, "erudite_footnotes_section") )
      removeById(doc, "erudite_footnotes");
  }

  private int setAByClass( final Document doc, final String clsname,
      final String href )
  {
    return setAByClass(doc, clsname, href, href);
  }

  private int setAByClass( final Document doc, final String clsname,
      final String href, final String text )
  {
    final Elements elements = doc.getElementsByClass(clsname);
    if ( elements == null )
      return 0;
    for ( final Element element : elements )
    {
      element.attr("href", href);
      if ( text != null )
      {
        element.empty();
        element.text(text);
      }
    }
    return elements.size();
  }

  @SuppressWarnings( "unused" )
  private boolean setAById( final Document doc, final String id,
      final String href )
  {
    return setAById(doc, id, href, href);
  }

  private boolean setAById( final Document doc, final String id,
      final String href, final String text )
  {
    final Element element = doc.getElementById(id);
    if ( element == null )
      return false;
    element.attr("href", href);
    element.empty();
    element.text(text);
    return true;
  }

  private int setTextElementByClass( final Document doc, final String clsname,
      final String text )
  {
    final Elements elements = doc.getElementsByClass(clsname);
    if ( elements == null )
      return 0;
    for ( final Element element : elements )
    {
      element.empty();
      element.text(text);
    }
    return elements.size();
  }

  @SuppressWarnings( "unused" )
  private boolean setTextElementById( final Document doc, final String id,
      final String text )
  {
    final Element element = doc.getElementById(id);
    if ( element == null )
      return false;
    element.empty();
    element.text(text);
    return true;
  }
} public final class HNSearch
{
  private static class HNHits
  {
    final List<HNItem> hits;

    @SuppressWarnings( "unused" )
    public HNHits( @JsonProperty( "hits" ) final List<HNItem> hits )
    {
      this.hits = hits;
    }

    @Override
    public String toString()
    {
      return "HNHits [hits=" + hits + "]";
    }

  }

  private static class HNItem implements Comparable<HNItem>
  {
    final String objectID;
    final String url;
    final int created_at_i;
    final int num_comments;

    @SuppressWarnings( "unused" )
    public HNItem( @JsonProperty( "objectID" ) final String objectID,
        @JsonProperty( "url" ) final String url //
        , @JsonProperty( "created_at_i" ) final int created_at_i //
        , @JsonProperty( "num_comments" ) final int num_comments //
    )
    {
      this.objectID = objectID;
      this.url = url;
      this.created_at_i = created_at_i;
      this.num_comments = num_comments;
    }

    @Override
    public int compareTo( final HNItem rhs )
    {
      int ret = rhs.num_comments - num_comments;
      if ( ret == 0 )
      {
        ret = created_at_i - rhs.created_at_i;
        if ( ret == 0 )
          ret = Str.ifNull(objectID).compareTo(Str.ifNull(rhs.objectID));
      }
      return ret;
    }

    @Override
    public boolean equals( final Object obj )
    {
      if ( this == obj )
        return true;
      if ( obj == null )
        return false;
      if ( getClass() != obj.getClass() )
        return false;
      HNItem other = (HNItem) obj;
      if ( created_at_i != other.created_at_i )
        return false;
      if ( num_comments != other.num_comments )
        return false;
      if ( objectID == null )
      {
        if ( other.objectID != null )
          return false;
      }
      else if ( !objectID.equals(other.objectID) )
        return false;
      if ( url == null )
      {
        if ( other.url != null )
          return false;
      }
      else if ( !url.equals(other.url) )
        return false;
      return true;
    }

    @Override
    public int hashCode()
    {
      final int prime = 31;
      int result = 1;
      result = prime * result + created_at_i;
      result = prime * result + num_comments;
      result = prime * result + ((objectID == null) ? 0 : objectID.hashCode());
      result = prime * result + ((url == null) ? 0 : url.hashCode());
      return result;
    }

    @Override
    public String toString()
    {
      return "HNItem [objectID=" + objectID + ", url=" + url + "]";
    }
  }

  private static class Source
  {
    final String sourceUrl;
    final String title;

    Source( final String source_url, final String title )
    {
      this.sourceUrl = source_url;
      this.title = title;
    }

    @Override
    public boolean equals( final Object obj )
    {
      if ( this == obj )
        return true;
      if ( obj == null )
        return false;
      if ( getClass() != obj.getClass() )
        return false;
      Source other = (Source) obj;
      if ( sourceUrl == null )
      {
        if ( other.sourceUrl != null )
          return false;
      }
      else if ( !sourceUrl.equals(other.sourceUrl) )
        return false;
      if ( title == null )
      {
        if ( other.title != null )
          return false;
      }
      else if ( !title.equals(other.title) )
        return false;
      return true;
    }

    @Override
    public int hashCode()
    {
      final int prime = 31;
      int result = 1;
      result = prime * result
          + ((sourceUrl == null) ? 0 : sourceUrl.hashCode());
      result = prime * result + ((title == null) ? 0 : title.hashCode());
      return result;
    }
  }

  public static final ImmutableList<String> EMPTY = ImmutableList.of();
  private static final LoadingCache<Source, ImmutableList<String>> cache = createCache();

  /**
   * Removes all lookups in the cache.
   */
  public static void clearCache()
  {
    cache.invalidateAll();
  }

  /**
   * Finds the URLs of the discussion threads on Hacker News for a web page.
   *
   * @param article
   *        The Article to look up.
   * @return The URLs of the discussion threads on Hacker News, sorted by most
   *         comments, or an empty list.
   */
  public static ImmutableList<String> lookup( final Article article )
  {
    try
    {
      return cache
          .get(new Source(article.getOriginalUrl(), article.getTitle()));
    }
    catch ( Exception ex )
    {
      LoggerFactory.getLogger(HNSearch.class)
          .trace("Error while HNSearch for " + article.getOriginalUrl(), ex);
      return ImmutableList.of();
    }
  }

  private static LoadingCache<Source, ImmutableList<String>> createCache()
  {
    return CacheBuilder.newBuilder().maximumSize(20)
        .build(new CacheLoader<Source, ImmutableList<String>>() {
          @Override
          public ImmutableList<String> load( final Source source )
          {
            final Logger log = LoggerFactory.getLogger(HNSearch.class);
            try
            {
              log.trace("hn.algolia.com lookup: {} {}", source.sourceUrl,
                source.title);
              ImmutableList<String> results = search(source.sourceUrl,
                source.sourceUrl);
              if ( results.isEmpty() && Str.isNotEmpty(source.title) )
              {
                log.trace(
                  "Could not find match based on URL, so searching on title instead.");
                results = search(source.title, source.sourceUrl);
              }
              if ( results.isEmpty() )
                log.trace("No Hacker News discussion thread for {}",
                  source.sourceUrl);
              return results;
            }
            catch ( Exception ex )
            {
              log.trace("Exception while looking up hn.algolia.com for: "
                  + source.sourceUrl,
                ex);
              return EMPTY;
            }
          }

          @SuppressWarnings( "synthetic-access" )
          private ImmutableList<String> search( final String term,
              final String source_url )
          {
            try
            {
              final String hnsearch_url = "http://hn.algolia.com/api/v1/search_by_date?tags=story&query="
                  + Esc.url.text(term);
              final Connection conn = Conn.connect(hnsearch_url);
              conn.ignoreContentType(true);
              final Response resp = conn.execute();
              if ( resp.statusCode() != 200 )
              {
                LoggerFactory.getLogger(HNSearch.class).trace(
                  "GET {} returned {}: " + resp.statusMessage(), hnsearch_url,
                  String.valueOf(resp.statusCode()));
                return EMPTY;
              }

              // Parse json return.
              final String body = resp.body();
              return parse(body, source_url);
            }
            catch ( IOException ex )
            {
              LoggerFactory.getLogger(HNSearch.class).trace(
                "Exception while looking up hn.algolia.com for: " + term, ex);
              return EMPTY;
            }
          }
        });
  }

  private static ImmutableList<String> parse( final String json,
      final String source_url )
    throws JsonParseException,
      IOException
  {
    final Logger log = LoggerFactory.getLogger(HNSearch.class);

    final ObjectMapper om = new ObjectMapper();
    om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true);
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    final HNHits hits = om.readValue(json, HNHits.class);
    final TreeMap<HNItem, String> items = new TreeMap<HNItem, String>();
    for ( final HNItem item : hits.hits )
      if ( source_url.equals(item.url) && Str.isNotEmpty(item.objectID) )
      {
        final String hn_url = "http://news.ycombinator.com/item?id="
            + item.objectID;
        final String hn_surl = "https://news.ycombinator.com/item?id="
            + item.objectID;
        if ( (!source_url.equalsIgnoreCase(hn_url))
            && (!source_url.equalsIgnoreCase(hn_surl)) )
        {
          log.trace("Hacker News discussion for {} is {}", source_url, hn_surl);
          items.put(item, hn_surl);
        }
      }
    if ( items.isEmpty() )
      return EMPTY;
    return ImmutableList.copyOf(items.values());
  }

  private HNSearch()
  {
    // empty
  }
} public final class ImageHandlerFactory
{
  private static class HadIOException extends IOException
  {
    public HadIOException( final String msg )
    {
      super(msg);
    }
  }

  /**
   * <p>
   * The contents of an image, plus the file name extension we should use.
   * </p>
   * <p>
   * If content is null, then:
   * </p>
   * <ul>
   * <li>if extension is also null then the image is unavailable and should be
   * removed from the article.</li>
   * <li>otherwise an exception occurred while retrieving the image, and
   * extension contains the message to go in the IOException.</li>
   * </ul>
   */
  private static class ImageContent
  {
    private final byte[] content;
    private final String name;
    private final String extension;

    ImageContent( final byte[] content, final String name,
        final String extension )
    {
      this.content = content;
      this.name = name;
      this.extension = extension;
    }

    public String getExtension()
    {
      return extension;
    }

    public String getName()
    {
      return name;
    }

    /**
     * Returns true if there is image content available to be saved.
     * 
     * @return True if there is image content available to be saved.
     */
    boolean isAvailable()
    {
      return content != null;
    }

    /**
     * Save a byte array to a file.
     * 
     * @param file
     * @param content
     * @throws IOException
     */
    void save( final File file ) throws IOException
    {
      if ( content == null )
        throw new IllegalStateException("There is no content for this image.");
      final OutputStream out = new FileOutputStream(file);
      try
      {
        out.write(content);
      }
      finally
      {
        out.close();
      }
    }

    /**
     * Throws an IOException if an exception occurred while retrieving the
     * image.
     * 
     * @throws HadIOException
     */
    void throwIfInError() throws HadIOException
    {
      if ( (content == null) && (extension != null) )
        throw new HadIOException(extension);
    }

    /**
     * Used by cache management.
     * 
     * @return Rough number of bytes this object uses.
     */
    int weight()
    {
      return Math.max(1, //
        ((content == null) ? 0 : content.length) //
            + (Str.length(name) * 2) //
            + (Str.length(extension) * 2));
    }
  }

  /**
   * Create factory with unlimited cache (and default concurrency).
   * 
   * @param null_on_error
   *        If there was an error loading an image, just return
   *        <code>null</code> instead of throwing an {@link IOException}.
   * @param convert_to_png
   *        Convert all downloaded images to PNG. This can reduce the
   *        possibility of weird images (such as animated gifs) from making a
   *        document unloadable by your ereader device.
   * @param min_width
   *        Minimum width of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param min_height
   *        Minimum height of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @return Image handler factory.
   */
  public static ImageHandlerFactory create( final boolean null_on_error,
      final boolean convert_to_png, final int min_width, final int min_height )
  {
    return new ImageHandlerFactory(null_on_error, convert_to_png, min_width,
        min_height, 0, -1L, -1L);
  }

  /**
   * Create factory with unlimited cache.
   * 
   * @param null_on_error
   *        If there was an error loading an image, just return
   *        <code>null</code> instead of throwing an {@link IOException}.
   * @param convert_to_png
   *        Convert all downloaded images to PNG. This can reduce the
   *        possibility of weird images (such as animated gifs) from making a
   *        document unloadable by your ereader device.
   * @param min_width
   *        Minimum width of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param min_height
   *        Minimum height of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param cache_concurrency
   *        Basically the expected number of threads that are expected to be
   *        accessing the image cache at once.
   * @return Image handler factory.
   */
  public static ImageHandlerFactory create( final boolean null_on_error,
      final boolean convert_to_png, final int min_width, final int min_height,
      final int cache_concurrency )
  {
    return new ImageHandlerFactory(null_on_error, convert_to_png, min_width,
        min_height, cache_concurrency, -1L, -1L);
  }

  /**
   * Create factory.
   * 
   * @param null_on_error
   *        If there was an error loading an image, just return
   *        <code>null</code> instead of throwing an {@link IOException}.
   * @param convert_to_png
   *        Convert all downloaded images to PNG. This can reduce the
   *        possibility of weird images (such as animated gifs) from making a
   *        document unloadable by your ereader device.
   * @param min_width
   *        Minimum width of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param min_height
   *        Minimum height of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param cache_concurrency
   *        Basically the expected number of threads that are expected to be
   *        accessing the image cache at once.
   * @param max_cache_members
   *        The maximum number of images to keep in the cache at once (
   *        <code>-1</code> for unlimited, do not specify a non-negative one
   *        value for both <code>max_cache_members</code> and
   *        <code>max_cache_mb</code> at the same time.)
   * @param max_cache_mb
   *        The maximum number of megabytes the cache should store (
   *        <code>-1L</code> for unlimited, do not specify a non-negative one
   *        value for both <code>max_cache_members</code> and
   *        <code>max_cache_mb</code> at the same time.)
   * @return Image handler factory.
   */
  public static ImageHandlerFactory create( final boolean null_on_error,
      final boolean convert_to_png, final int min_width, final int min_height,
      final int cache_concurrency, final long max_cache_members,
      final long max_cache_mb )
  {
    return new ImageHandlerFactory(null_on_error, convert_to_png, min_width,
        min_height, cache_concurrency, max_cache_members, max_cache_mb);
  }

  /**
   * Create an iamge factory based on settings in the configuation file (see
   * user documentation.)
   * 
   * @param config
   *        Configuration to use.
   * @return Image handler factory.
   */
  public static ImageHandlerFactory create( final Config config )
  {
    return create( //
      config.getInt("worker.threads", 1) //
      , config);
  }

  /**
   * Create an iamge factory based on settings in the configuation file (see
   * user documentation.)
   * 
   * @param cache_concurrency
   *        Basically the expected number of threads that are expected to be
   *        accessing the image cache at once.
   * @param config
   *        Configuration to use.
   * @return Image handler factory.
   */
  public static ImageHandlerFactory create( final int cache_concurrency,
      final Config config )
  {
    final int maxmembers = config.getInt("image.cache.max.members", -1);
    int maxmb = config.getInt("image.cache.max.mb", -1);
    if ( maxmembers >= 0 )
      maxmb = 0;
    else if ( maxmb < 0 )
      maxmb = 100;

    return new ImageHandlerFactory( //
        config.getBoolean("image.ignore.errors", false) //
        , config.getBoolean("image.to.png", false) //
        , config.getInt("image.min.width", 0) //
        , config.getInt("image.min.height", 0) //
        , Math.max(1, Math.min(20, cache_concurrency)) //
        , maxmembers //
        , maxmb //
    );
  }

  /**
   * Create factory that does not cache.
   * 
   * @param null_on_error
   *        If there was an error loading an image, just return
   *        <code>null</code> instead of throwing an {@link IOException}.
   * @param convert_to_png
   *        Convert all downloaded images to PNG. This can reduce the
   *        possibility of weird images (such as animated gifs) from making a
   *        document unloadable by your ereader device.
   * @param min_width
   *        Minimum width of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @param min_height
   *        Minimum height of an image in order to be included in the output.
   *        This is helpful to eliminate less useful things like web bugs.
   * @return Image handler factory.
   */
  public static ImageHandlerFactory createNoCache( final boolean null_on_error,
      final boolean convert_to_png, final int min_width, final int min_height )
  {
    return new ImageHandlerFactory(null_on_error, convert_to_png, min_width,
        min_height, 0, 0, -1L);
  }

  private final boolean nullOnError;
  private final boolean convertToPng;
  private final int minWidth;
  private final int minHeight;
  private final LoadingCache<String, ImageContent> cache;
  private final FilenameTransformer filenameTransformer = new FilenameTransformer(
      FilenameTransformer.NonAsciiHandling.ASCIIFY, 200);
  private final Logger log = LoggerFactory.getLogger(ImageHandlerFactory.class);
  private final Pattern RECURSIVE_URL = Pattern
      .compile("^https?:\\/\\/.+%5[cC]%22(https?:\\/\\/.+)%5[cC]%22$");

  private ImageHandlerFactory( final boolean null_on_error,
      final boolean convert_to_png, final int min_width, final int min_height,
      final int cache_concurrency, final long max_cache_members,
      final long max_cache_mb )
  {
    this.nullOnError = null_on_error;
    this.convertToPng = convert_to_png;
    this.minWidth = Math.max(0, min_width);
    this.minHeight = Math.max(0, min_height);

    final CacheBuilder<String, ImageContent> bldr = CacheBuilder.newBuilder()
        .softValues()
        .removalListener(new RemovalListener<String, ImageContent>() {
          @SuppressWarnings( "synthetic-access" )
          @Override
          public void onRemoval(
              final RemovalNotification<String, ImageContent> event )
          {
            final String cause;
            switch ( event.getCause() )
            {
              case COLLECTED:
                cause = "its memory was released";
                break;
              case EXPIRED:
                cause = "it had expired";
                break;
              case EXPLICIT:
                cause = "it was manually removed";
                break;
              case REPLACED:
                cause = "it was replaced with a new value";
                break;
              case SIZE:
                cause = "space was needed";
                break;
              default:
                cause = event.getCause().toString();
                break;
            }
            log.trace("Removed image from cache because {}: {}", cause,
              event.getKey());
          }
        });

    if ( cache_concurrency >= 0 )
      bldr.concurrencyLevel(Math.min(20, cache_concurrency));
    if ( max_cache_members >= 0 )
      bldr.maximumSize(max_cache_members);
    if ( max_cache_mb >= 0 )
    {
      bldr.maximumWeight(Math.min(1024L, max_cache_mb) * 1024L * 1024L);
      bldr.weigher(new Weigher<String, ImageContent>() {
        @Override
        public int weigh( final String url, final ImageContent image )
        {
          return image.weight() + (url.length() * 2);
        }
      });
    }

    cache = bldr.build(new CacheLoader<String, ImageContent>() {
      @SuppressWarnings( "synthetic-access" )
      @Override
      public ImageContent load( final String url )
      {
        try
        {
          log.trace("Retriving image: {}", url);
          final ImageContent image;
          {
            // Retrieve image content from URL.
            final Connection conn = Conn.connect(url);
            conn.ignoreContentType(true);
            final Response resp = conn.execute();
            if ( resp.statusCode() != 200 )
              throw new IOException("GET " + url + " returned "
                  + resp.statusCode() + ": " + resp.statusMessage());
            final byte[] content = resp.bodyAsBytes();
            if ( Arr.isEmpty(content) )
              throw new IOException("GET " + url + " returned zero bytes.");

            String name = getName(url);
            String ext = getExt(url);
            if ( ext == null )
            {
              ext = getExt(resp);
              if ( ext == null )
              {
                ext = getExt(content);
                if ( ext == null )
                  throw new IOException("Unknown extension for URL " + url);
              }
            }
            image = new ImageContent(content //
                , filenameTransformer.transform(name, "image") //
                , filenameTransformer.transform(ext, ".image").toLowerCase());
          }

          if ( (minWidth > 0) || (minHeight > 0) || convertToPng )
          {
            final BufferedImage bimage;
            final InputStream in = new ByteArrayInputStream(image.content);
            try
            {
              bimage = ImageIO.read(in);
            }
            finally
            {
              in.close();
            }

            if ( bimage == null )
              throw new NullPointerException("Doesn't appear to be an image.");

            if ( (bimage.getWidth() <= minWidth)
                || (bimage.getHeight() <= minHeight) )
            {
              log.trace("Image too small to be used: {}", url);
              return new ImageContent(null, null, null);
            }

            if ( convertToPng )
            {
              final ImageContent png = convert(bimage, image.getName());
              log.trace("Retrieved image and converted to PNG: {}", url);
              return png;
            }
          }

          log.trace("Retrieved image: {}", url);
          return image;
        }
        catch ( Exception ex )
        {
          log.trace("Exception while retriving image: " + url, ex);
          return new ImageContent(null, null, ex.getMessage());
        }
      }

      private ImageContent convert( final BufferedImage image, final String name )
        throws IOException
      {
        final ByteArrayOutputStream out = new ByteArrayOutputStream();
        try
        {
          ImageIO.write(image, "png", out);
        }
        finally
        {
          out.close();
        }
        return new ImageContent(out.toByteArray(), name, ".png");
      }

      /**
       * Make sure name is all lower-case, starts with a letter, and only
       * contains ASCII letters and numbers.
       * 
       * @param orig
       * @return
       */
      private String cookName( final String orig )
      {
        final Transform lower = new Transform() {
          @Override
          public String transform( final char ch )
          {
            final char newch = Character.toLowerCase(ch);
            if ( newch == ch )
              return null;
            return String.valueOf(newch);
          }
        };

        final Transform stripstd = new Transform() {
          @Override
          public String transform( final char ch )
          {
            if ( (ch < 'a') || (ch > 'z') )
              if ( (ch < 'A') || (ch > 'Z') )
                if ( (ch < '0') || (ch > '9') )
                  return Str.EMPTY; // Remove
            return null;
          }
        };

        final Transform stripinitial = new Transform() {
          @Override
          public String transform( final char ch )
          {
            if ( (ch < 'a') || (ch > 'z') )
              if ( (ch < 'A') || (ch > 'Z') )
                return Str.EMPTY; // Remove
            return null;
          }
        };

        final Transform initial = new AndTransform(TransformAsciify.INSTANCE,
            stripinitial, lower);
        final Transform std = new AndTransform(TransformAsciify.INSTANCE,
            stripstd, lower);

        final int len = orig.length();
        final StringBuilder buff = new StringBuilder(len);
        int idx = 0;
        while ( idx < len )
        {
          final char ch = orig.charAt(idx);
          final String str = initial.transform(ch);
          if ( str == null )
            buff.append(ch);
          else
            buff.append(str);
          ++idx;
          if ( buff.length() > 0 )
            break;
        }
        while ( idx < len )
        {
          final char ch = orig.charAt(idx);
          final String str = std.transform(ch);
          if ( str == null )
            buff.append(ch);
          else
            buff.append(str);
          ++idx;
        }
        return Str.ifEmpty(buff.toString(), null);
      }

      private String getExt( final byte[] content )
      {
        final MimeUtil2 mu = new MimeUtil2();
        mu.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector");

        final Collection<?> mimetypes = mu.getMimeTypes(content);
        if ( (mimetypes == null) || mimetypes.isEmpty() )
          return null;
        final MimeType mt = MimeUtil2.getMostSpecificMimeType(mimetypes);
        if ( (mt == null) || MimeUtil2.UNKNOWN_MIME_TYPE.equals(mt) )
          return null;

        if ( !"image".equals(mt.getMediaType()) )
          return null;

        final String ext = cookName(mt.getSubType());
        if ( Str.isEmpty(ext) )
          return null;

        return "." + ext;
      }

      private String getExt( final Response resp )
      {
        final String mimetype = resp.header("Content-Type");
        final int end;
        {
          final int pos = mimetype.indexOf(' ');
          end = (pos > 0) ? pos : mimetype.length();
        }
        if ( (end <= 7) || (!mimetype.startsWith("image/")) )
          return null;

        final String ext = cookName(mimetype.substring(6, end));
        if ( Str.isEmpty(ext) )
          return null;

        return "." + ext;
      }

      private String getExt( final String source )
      {
        final int end = getUrlEnd(source);
        final String ext = cookName(FileName.extension(source.substring(0, end)));
        final int extlen = Str.length(ext);
        if ( (extlen > 0) && (extlen <= 3) )
          return "." + ext;
        return null;
      }

      private String getName( final String source )
      {
        final int end = getUrlEnd(source);
        return Str.ifEmpty(cookName(FileName.sansExtension(FileName
            .baseName(source.substring(0, end)))), "image");
      }

      private int getUrlEnd( final String url )
      {
        if ( url == null )
          return 0;
        int pos = url.indexOf('?');
        if ( pos > 0 )
          return pos;
        return url.length();
      }
    });
  }

  /**
   * Removes all images in the cache.
   */
  public void clearCache()
  {
    cache.invalidateAll();
  }

  /**
   * Get an image handler that uses the cache provided by this factory.
   * 
   * @param folder
   *        Where to save the image file. Will always create a file with a
   *        unique name.
   * @param prefix
   *        A prefix to be added to the name of the file that is returned by
   *        {@link ImageHandler#image(String)}. This will usually represent the
   *        path to the image file relative to the HTML file being produced.
   * @return A new image handler.
   */
  public ImageHandler get( final File folder, final String prefix )
  {
    return new ImageHandler() {
      @SuppressWarnings( "synthetic-access" )
      @Override
      public String image( final String source ) throws IOException
      {
        final String url = sourceToUrl(source);
        if ( url == null )
        {
          log.trace("Not a valid URL for an image: {}", source);
          return null;
        }

        try
        {
          final ImageContent image = cache.get(url);
          if ( !image.isAvailable() )
          {
            image.throwIfInError();
            return null;
          }

          final File file;
          final String path;
          {
            final UniqueFile uf = new UniqueFile(folder, image.getName(),
                image.getExtension());
            file = uf.getFile();
            path = (prefix == null) ? file.getName()
                : (prefix + file.getName());
          }

          if ( !folder.exists() )
            Folders.mks(folder);

          image.save(file);

          log.trace("Saved image {} to {}", url, file);
          return path;
        }
        catch ( HadIOException ex )
        {
          if ( nullOnError )
            return null;
          throw new IOException("Error retrieving " + url, ex);
        }
        catch ( Exception ex )
        {
          throw new IOException("Error processing " + url, ex);
        }
      }

      private String sourceToUrl( final String source )
      {
        if ( (Str.length(source) < 8)
            || ((!source.startsWith("http://")) && (!source
                .startsWith("https://"))) )
        {
          // Not a valid HTTP protocol URL, skip it.
          return null;
        }

        {
          // Occasionally seem to get this funny URL in a URL situation
          // (especially with articles from readwriteweb.com. Think it's
          // something with the way Instapaper saves the text. If this happens,
          // just call ourself with the inner URL.
          @SuppressWarnings( "synthetic-access" ) final Matcher mat = RECURSIVE_URL
              .matcher(source);
          if ( mat.matches() )
            return sourceToUrl(mat.group(1));
        }

        return source;
      }
    };
  }
} public interface Processor
{
  /**
   * Processes an article. See the user documentation (in particular, the sample
   * configuration file) for what this can do.
   * 
   * @param article
   *        The article to process.
   * @param erudite
   *        An {@link Erudite} object for formatting the HTML document.
   * @param source
   *        The source of the article.
   * @param ihf
   *        An image factory handler.
   * @param work_folder
   *        A temporary folder that can use to do all its processing.
   * @throws Exception
   */
  void process( Article article, Erudite erudite, Source source,
      ImageHandlerFactory ihf, File work_folder ) throws Exception;
} public interface Source
{
  /**
   * List of articles to process.
   *
   * @return List of articles to process.
   * @throws IOException
   */
  Articles getArticles() throws IOException;

  /**
   * The name of the source.
   *
   * @return The name of the source.
   */
  String getName();

  /**
   * Little piece of HTML that is plugged into the footer of our documents to
   * indicate what source we retrieved the articles via.
   *
   * @return Small HTML string indicating the source we retrieved the artices
   *         via.
   */
  String getViaHtml();

  /**
   * Perform an action on an article that was successfully processed (e.g.,
   * delete it from the source, archive it, move it to another folder.)
   *
   * @param article
   *        Article to post-process.
   * @throws IOException
   */
  void onComplete( Article article ) throws IOException;

  /**
   * Perform an action on an article when an error occured while processing it
   * (e.g., move it to another folder to be looked at later.)
   *
   * @param article
   *        Article to post-process.
   * @throws IOException
   */
  void onError( Article article ) throws IOException;
} public class Template
{
  private final Document doc;

  public Template( final Document doc )
  {
    this.doc = doc.clone();
  }

  public Template( final File file ) throws IOException
  {
    this(Files.asByteSource(file));
  }

  public Template( final ByteSource ins ) throws IOException
  {
    final InputStream in = ins.openStream();
    try
    {
      this.doc = Jsoup.parse(in, "UTF-8", Str.EMPTY);
    }
    finally
    {
      in.close();
    }
  }

  /**
   * Retrieves a copy of the HTML document. The calling code can modify this
   * without effecting the original template.
   *
   * @return A copy of the HTML document for this template.
   */
  public Document getDocument()
  {
    return doc.clone();
  }
} public class TemplateFactory
{
  private static final String DEFAULT_TEMPLATE_KEY = "__erudite_default_template_key";

  private final ConcurrentHashMap<String, Template> templates = new ConcurrentHashMap<String, Template>();
  private final ByteSource defaultTemplateSupplier;
  private final Object[] lock = new Object[0];

  /**
   * Construct the template factory.
   *
   * @param default_template
   *        Input supplier for loading the default template (if needed).
   * @see #getDefault()
   */
  public TemplateFactory( final ByteSource default_template )
  {
    this.defaultTemplateSupplier = default_template;
  }

  /**
   * Loads the {@link Template} from the specified HTML file. If
   * <code>file</code> is <code>null</code> then same as calling
   * {@link #getDefault()}.
   *
   * @param file
   * @return The template.
   * @throws IOException
   */
  public Template get( final File file ) throws IOException
  {
    if ( file == null )
      return getDefault();
    final String path = Files.getCanonicalPath(file);
    Template template = templates.get(path);
    if ( template == null )
      synchronized ( lock )
      {
        template = templates.get(path);
        if ( template == null )
        {
          template = new Template(file);
          templates.put(path, template);
          LoggerFactory.getLogger(getClass()).trace("Loaded template from: {}",
            file);
        }
      }
    return template;
  }

  /**
   * Loads the {@link Template} from the specified HTML file. If
   * <code>filename</code> is an empty string or <code>null</code> then same as
   * calling {@link #getDefault()}. If the string contains just the name of a
   * file (no folder info) it will look for the file in the current directory
   * and failing that, the erudite application folder (e.g.,
   * <code>~/.erudite</code>).
   *
   * @param filename
   * @return The template.
   * @throws IOException
   */
  public Template get( final String filename ) throws IOException
  {
    if ( Str.isEmpty(filename) )
      return getDefault();

    File file = new File(filename);
    if ( !file.exists() )
      if ( Str.equals(file.getName(), file.getPath()) )
      {
        File alt = new File(Args.defUserDataFolder(), file.getName());
        if ( alt.exists() )
          file = alt;
      }

    return get(file);
  }

  /**
   * Loads the default template. This will either be the internally stored
   * template, or the template for the session file (if it exists).
   *
   * @return The default template.
   * @throws IOException
   */
  public Template getDefault() throws IOException
  {
    Template template = templates.get(DEFAULT_TEMPLATE_KEY);
    if ( template == null )
      synchronized ( lock )
      {
        template = templates.get(DEFAULT_TEMPLATE_KEY);
        if ( template == null )
        {
          template = new Template(defaultTemplateSupplier);
          templates.put(DEFAULT_TEMPLATE_KEY, template);
          LoggerFactory.getLogger(getClass()).trace("Loaded default template.");
        }
      }
    return template;
  }
} public interface Config
{
  /**
   * Retrieve a boolean configuration variable. One of (case insenstive) &ldquo;
   * <code>true</code> &rdquo;, &ldquo;<code>yes</code>&rdquo;, &ldquo;
   * <code>y</code>&rdquo; or &ldquo;<code>1</code>&rdquo; for true, or &ldquo;
   * <code>false</code>&rdquo;, &ldquo;<code>no</code>&rdquo;, &ldquo;
   * <code>n</code>&rdquo; or &ldquo; <code>0</code>&rdquo; for false.
   * 
   * @param key
   *        The key to look-up.
   * @param def
   *        What to return if the key is unspecified or empty.
   * @return The value or the default.
   * @throws IllegalStateException
   *         Thrown if the value is not one of the above valid values.
   */
  boolean getBoolean( String key, boolean def );

  /**
   * Retrieve a integer configuration variable.
   * 
   * @param key
   *        The key to look-up.
   * @param def
   *        What to return if the key is unspecified or empty.
   * @return The value or the default.
   * @throws IllegalStateException
   *         Thrown if the value is not one of the above valid values.
   */
  int getInt( String key, int def );

  /**
   * Return a string configuration variable. Equivalent to
   * <code>get(key, null)</code>.
   * 
   * @param key
   *        The key to look-up.
   * @return The value, or <code>null</code> if the key is unspecified or empty.
   */
  String getString( String key );

  /**
   * Return a string configuration variable.
   * 
   * @param key
   *        The key to look-up.
   * @param def
   *        What to return if the key is unspecified or empty (may be
   *        <code>null</code>).
   * @return The value or the default.
   */
  String getString( String key, String def );

  /**
   * An array of strings from a key that is specified zero or more times within
   * the configuration file.
   * 
   * @param key
   *        The key to look-up.
   * @return A list of the values (will be an empty list if the key is
   *         unspecified, empty values are ignored).
   */
  ImmutableList<String> getStrings( String key );

  /**
   * The title munger, built from the values of
   * <code>getStrings(&quot;title&quot;)</code>. See the sample configuration
   * file in the user documentation.
   * 
   * @return The title munger.
   */
  TitleMunger getTitleMunger();
} public final class ConfigUtils
{
  /**
   * Return the value of the first key specified in the configuration.
   * 
   * @param config
   * @param key1
   * @param key2
   * @return The value of the first key specified in the configuration, or
   *         <code>null</code>.
   */
  public static String getFirst( final Config config, final String key1,
      final String key2 )
  {
    String ret = config.getString(key1);
    if ( ret == null )
      ret = config.getString(key2);
    return ret;
  }

  /**
   * Return the value of the first key specified in the configuration.
   * 
   * @param config
   * @param key1
   * @param key2
   * @param key3
   * @return The value of the first key specified in the configuration, or
   *         <code>null</code>.
   */
  public static String getFirst( final Config config, final String key1,
      final String key2, final String key3 )
  {
    String ret = getFirst(config, key1, key2);
    if ( ret == null )
      ret = config.getString(key3);
    return ret;
  }

  /**
   * Return the value of the first key specified in the configuration.
   * 
   * @param config
   * @param key1
   * @param key2
   * @param key3
   * @param key4
   * @return The value of the first key specified in the configuration, or
   *         <code>null</code>.
   */
  public static String getFirst( final Config config, final String key1,
      final String key2, final String key3, final String key4 )
  {
    String ret = getFirst(config, key1, key2, key3);
    if ( ret == null )
      ret = config.getString(key4);
    return ret;
  }

  /**
   * Convert a string to a boolean value. One of (case insenstive) &ldquo;
   * <code>true</code> &rdquo;, &ldquo;<code>yes</code>&rdquo;, &ldquo;
   * <code>y</code>&rdquo; or &ldquo;<code>1</code>&rdquo; for true, or &ldquo;
   * <code>false</code>&rdquo;, &ldquo;<code>no</code>&rdquo;, &ldquo;
   * <code>n</code>&rdquo; or &ldquo; <code>0</code>&rdquo; for false.
   * 
   * @param str
   *        The string to convert.
   * @param def
   *        What to return if the string is empty or <code>null</code>.
   * @return The value or the default.
   * @throws IllegalStateException
   *         Thrown if the value is not one of the above valid values.
   */
  public static boolean toBoolean( final String str, final boolean def )
  {
    if ( Str.isEmpty(str) )
      return def;
    if ( Str.equalsOneOfIgnoreCase(str, "true", "y", "1", "yes") )
      return true;
    if ( Str.equalsOneOfIgnoreCase(str, "false", "n", "0", "no") )
      return false;
    throw new IllegalStateException("Invalid boolean value: " + str);
  }

  private ConfigUtils()
  {
    // empty
  }
} public final class Doc
{
  public static void write( final Document doc, final File file )
    throws IOException
  {
    final Writer out = new UTF8FileWriter(file);
    try
    {
      write(doc, out);
    }
    finally
    {
      out.close();
    }
  }

  public static void write( final Document doc, final OutputStream out )
    throws IOException
  {
    final Writer wout = new OutputStreamWriter(out, Charsets.UTF8);
    write(doc, wout);
    wout.flush();
  }

  public static void write( final Document doc, final Writer out )
    throws IOException
  {
    out.write(doc.outerHtml());
  }

  private Doc()
  {
    // empty
  }
} public final class EpubWrapHack
{
  private static final String MAGIC = "@@ERUDITE@@WRAP@@HACK@@";
  private static final byte[] MAGIC_BYTES = MAGIC.getBytes(Charsets.UTF8);
  private static final byte[] ZERO_WIDTH_BYTES = "\u200B"
      .getBytes(Charsets.UTF8);

  /**
   * Process the produced epub file, swapping our special marker for zero width
   * spaces.
   * 
   * @param from
   *        epub file containing our special markers.
   * @param to
   *        epub file to be written with zero width spaces inserted.
   * @throws IOException
   */
  public static void postProcess( final File from, final File to )
    throws IOException
  {
    final InputStream in = new FileInputStream(from);
    try
    {
      final OutputStream out = new FileOutputStream(to);
      try
      {
        postProcess(in, out);
      }
      finally
      {
        out.close();
      }
    }
    finally
    {
      in.close();
    }
  }

  /**
   * Pre-process our HTML document, looking for elements with the class &ldquo;
   * <code>erudite_url_url</code>&rdquo; and inserting our special marker in the
   * contained text at the points were we want to allow word wrapping.
   * 
   * @param orig
   *        The document to process.
   * @return A copy of <code>orig</code> with our markers inserted.
   */
  public static Document preProcess( final Document orig )
  {
    LoggerFactory.getLogger(EpubWrapHack.class).trace(
      "Performing Epub Wrap Hack pre-processing.");
    final Document doc = orig.clone();
    for ( Element el : doc.getElementsByClass("erudite_url_url") )
      hack(el);
    return doc;
  }

  private static void copy( final InputStream in, final OutputStream out,
      final byte[] buff ) throws IOException
  {
    int len;
    while ( (len = in.read(buff)) >= 0 )
      out.write(buff, 0, len);
  }

  private static int findMagic( final byte[] buff, final int start,
      final int end )
  {
    final int max = end - MAGIC_BYTES.length + 1;
    outer: for ( int xi = start; xi < max; ++xi )
    {
      for ( int xj = 0; xj < MAGIC_BYTES.length; ++xj )
        if ( buff[xi + xj] != MAGIC_BYTES[xj] )
          continue outer;
      return xi;
    }
    return -1;
  }

  private static boolean hack( final Element element )
  {
    final StringBuilder text = new StringBuilder(element.text());
    boolean changed = false;
    for ( int xi = text.length() - 3; xi > 10; --xi )
      switch ( text.charAt(xi) )
      {
        case '/':
        case '=':
        case '&':
        case '?':
          text.insert(xi, MAGIC);
          changed = true;
          break;
      }
    if ( changed )
      element.text(text.toString());
    return changed;
  }

  private static void postProcess( final InputStream from, final OutputStream to )
    throws IOException
  {
    final ZipInputStream in = new ZipInputStream(from);
    final ZipOutputStream out = new ZipOutputStream(to);
    try
    {
      out.setLevel(9);
      postProcess(in, out);
    }
    finally
    {
      out.finish();
    }
  }

  private static void postProcess( final ZipInputStream from,
      final ZipOutputStream to ) throws IOException
  {
    LoggerFactory.getLogger(EpubWrapHack.class).trace(
      "Performing Epub Wrap Hack post-processing.");
    final byte[] buff = new byte[1024 * 1024];
    ZipEntry inentry;
    while ( (inentry = from.getNextEntry()) != null )
    {
      final boolean is_html = Str
          .endsWithIgnoreCase(inentry.getName(), ".html");
      final ZipEntry outentry = new ZipEntry(inentry.getName());
      to.putNextEntry(outentry);
      try
      {
        if ( is_html )
          unhack(from, to, buff);
        else
          copy(from, to, buff);
      }
      finally
      {
        to.closeEntry();
      }
    }
  }

  private static void unhack( final InputStream in, final OutputStream out,
      final byte[] buff ) throws IOException
  {
    int pos = 0;
    boolean eof = false;
    while ( !eof )
    {
      final int cnt = in.read(buff, pos, buff.length - pos);
      eof = cnt < 0;
      final int end = (cnt <= 0) ? pos : pos + cnt;
      final int lastpos = (cnt < 0) ? end //
          : Math.max(0, end - MAGIC_BYTES.length + 1);
      pos = 0;
      while ( pos < lastpos )
      {
        int idx = findMagic(buff, pos, end);
        if ( idx < 0 )
        {
          if ( pos < lastpos )
          {
            out.write(buff, pos, lastpos - pos);
            pos = lastpos;
          }
        }
        else
        {
          if ( pos < idx )
            out.write(buff, pos, idx - pos);
          out.write(ZERO_WIDTH_BYTES);
          pos = idx + MAGIC_BYTES.length;
        }
      }

      if ( pos >= end )
      {
        pos = 0;
      }
      else
      {
        final int len = end - pos;
        System.arraycopy(buff, pos, buff, 0, len);
        pos = len;
      }
    }
  }

  private EpubWrapHack()
  {
    // empty
  }
} public final class Utils
{
  /**
   * Are we running on Windows?
   */
  public static final boolean IS_WINDOWS = Str
      .ifNull(System.getProperty("os.name")).toLowerCase().indexOf("windows") >= 0;

  private static String[] _paths;

  /**
   * If the string is too long, then abbreviate it to an excerpt with a trailing
   * ellipsis.
   * 
   * @param text
   *        String to abbreviate if necessary.
   * @param maxlen
   *        Maximum length of the returned string.
   * @return Abbreviated string.
   */
  public static String abbreviate( final String text, final int maxlen )
  {
    final String str = Str.trimToEmpty(text);
    if ( Str.length(str) <= maxlen )
      return str;

    // Find the end of a word.
    final int minlen = Math.max(1, maxlen / 2);
    int pos = maxlen - 1;
    while ( (pos >= minlen) && (!Character.isWhitespace(str.charAt(pos))) )
      --pos;
    while ( (pos >= minlen)
        && (!Character.isLetterOrDigit(str.charAt(pos - 1))) )
      --pos;
    if ( pos < minlen )
      pos = maxlen - 1;
    return str.substring(0, pos) + '\u2026';
  }

  /**
   * Find an existing file an array of paths.
   * 
   * @param name
   *        The name of the file we are looking for.
   * @param paths
   *        The paths to search.
   * @return The existing file in the first matching path, or <code>null</code>
   *         if not found.
   */
  public static File findInFolder( final String name, final String... paths )
  {
    for ( final String path : paths )
    {
      final File file = new File(path, name);
      if ( file.exists() )
        return Files.getCanonicalFile(file);
    }
    return null;
  }

  /**
   * Searches for an existing file in the folders specified by the operating
   * system's PATH environment variable.
   * 
   * @param name
   *        The name of the file we are looking for.
   * @return The existing file in the first matching path, or <code>null</code>
   *         if not found.
   */
  public static File findOnPath( final String name )
  {
    return findInFolder(name, getPaths());
  }

  /**
   * Create a HTML slug summary of the article. This basically looks for the
   * first paragraph tag, then starts converting each paragraph to text until it
   * gets about 1,000 characters.
   * 
   * @param source
   *        The source of the article.
   * @param article
   *        The article to summarise.
   * @param hacker_news_urls
   *        The hacker news discussion URLs for the article if available.
   * @return The HTML summary.
   */
  public static String summary( final Source source, final Article article,
      final ImmutableList<String> hacker_news_urls )
  {
    // IMPORTANT NOTE: Usage single quotes instead of double-quotes in the HTML
    // code you generate. When using the summary as a command line argument
    // under windows, the double-quotes tends to break things.

    final int MAXLEN = 1000;

    String summary = abbreviate(article.getSummary(), MAXLEN);
    if ( Str.isNotEmpty(summary) )
    {
      summary = "<p>" + Esc.htmlFull.text(summary) + "</p>";
    }
    else
    {
      try
      {
        Element el = null;
        // Find first paragraph element.
        {
          Iterator<Element> it = article.text().children().iterator();
          OUTER: while ( it.hasNext() )
          {
            final Element top = it.next();
            if ( top.tagName().equalsIgnoreCase("p") )
            {
              if ( top.hasText() )
              {
                el = top;
                break;
              }
            }
            else
            {
              for ( Element sub : top.getElementsByTag("p") )
                if ( (sub != null) && sub.hasText() )
                {
                  el = sub;
                  break OUTER;
                }
            }
          }
        }

        // Now run through current and following elements to build up the
        // summary text.
        final StringBuilder buff = new StringBuilder(MAXLEN + 100);
        int lentogo = MAXLEN;
        final int MINLEN = 100;
        while ( (el != null) && (lentogo > MINLEN) )
        {
          if ( el.hasText() )
          {
            final String text = Utils.abbreviate(el.text(), lentogo);
            buff.append("<p>").append(Esc.htmlFull.text(text)).append("</p>");
            lentogo -= text.length();
            if ( lentogo <= MINLEN )
              break;
          }

          Element next = el.nextElementSibling();
          while ( next == null )
            try
            {
              if ( el != null )
                el = el.parent();
              if ( (el == null) || (el.parent() == null) )
                break;
              next = el.nextElementSibling();
            }
            catch ( NullPointerException ex )
            {
              // Sometimes we get an unexpected null point exception from the
              // jsoup library, in which case we just run with what we have.
              next = null;
            }
          el = next;
        }
        summary = buff.toString();
      }
      catch ( IOException ex )
      {
        summary = Str.EMPTY;
      }
    }

    // IMPORTANT NOTE: Usage single quotes instead of double-quotes in the HTML
    // code you generate. When using the summary as a command line argument
    // under windows, the double-quotes tends to break things.

    final String original_url = article.getOriginalUrl();
    final String source_url = article.getSourceUrl();
    final StringBuilder buff = new StringBuilder();

    buff.append("<p><a href='").append(Esc.htmlFull.attr(original_url))
        .append("'>").append(Esc.htmlFull.text(original_url)).append("</a>");

    if ( Str.isNotEmpty(source_url) )
      buff.append("<span style='font-size: 80%'><br><a href='")
          .append(Esc.htmlFull.attr(source_url)).append("'>Read at ")
          .append(Esc.htmlFull.text(source.getName())).append(".</a></span>");

    if ( (hacker_news_urls != null) && (!hacker_news_urls.isEmpty()) )
    {
      buff.append("<span style='font-size: 80%'><br>(");
      if ( hacker_news_urls.size() == 1 )
      {
        buff.append("<a href='")
            .append(Esc.htmlFull.attr(hacker_news_urls.get(0)))
            .append("'>Hacker News discussion.</a>");
      }
      else
      {
        buff.append("Hacker News discussions: ");
        int link_num = 0;
        for ( final String hn_url : hacker_news_urls )
        {
          if ( link_num > 0 )
            buff.append(", ");
          buff.append(" <a href='").append(Esc.htmlFull.attr(hn_url))
              .append("'>#").append(++link_num).append("</a>");
        }
        buff.append('.');
      }
      buff.append(")</span>");
    }

    buff.append("</p>");
    if ( Str.isNotEmpty(summary) )
      buff.append(summary);
    return buff.toString();
  }

  private static String[] getPaths()
  {
    String[] paths = _paths;
    if ( paths == null )
    {
      final String pathstr = System.getenv("PATH");
      if ( Str.isEmpty(pathstr) )
      {
        paths = new String[0];
      }
      else
      {
        final String[] arr = pathstr.split(IS_WINDOWS ? ";" : ":");
        final ArrayList<String> list = new ArrayList<String>(arr.length);
        for ( String str : arr )
          if ( Str.isNotEmpty(str) )
            list.add(str);
        paths = list.toArray(new String[list.size()]);
      }
      _paths = paths;
    }
    return paths;
  }

  private Utils()
  {
    // empty
  }
} public interface Reservation
{
  /**
   * Removes the file and/or folder that were reserved. Call this if there was
   * an error while processing the article or when calling
   * {@link Then#then(Reservation, File, File)}.
   */
  void cleanup();

  /**
   * The base name (sans suffixes) of the unique file and/or folder that has
   * been reserved.
   * 
   * @return The base name (sans suffixes) of the unique file and/or folder that
   *         has been reserved.
   */
  String getBaseName();

  /**
   * The file that was reserved or <code>null</code>.
   * 
   * @return The file that was reserved or <code>null</code>.
   */
  File getFile();

  /**
   * The name of the file that was reserved or <code>null</code>.
   * 
   * @return The name of the file that was reserved or <code>null</code>.
   */
  String getFileName();

  /**
   * The folder that was reserved or <code>null</code>.
   * 
   * @return The folder that was reserved or <code>null</code>.
   */
  File getFolder();

  /**
   * The name of the folder that was reserved or <code>null</code>.
   * 
   * @return The name of the folder that was reserved or <code>null</code>.
   */
  String getFolderName();
} public interface Then
{
  /**
   * Reserve a unique file and/or folder for saving the processed document.
   * Sometimes we need to know the final name of the file and/or folder to be
   * produced before we actually commence processing the article.
   * 
   * @param article
   *        The article we are creating the file and/or folder for. The name of
   *        the file and/or folder will be based on the title of the article.
   * @param file_suffix
   *        The suffix to add to the name of the file (e.g., &ldquo;
   *        <code>.txt</code>&rdquo;.) If <code>null</code> then no file is
   *        created.
   * @param folder_suffix
   *        The suffix to add to the name of the folder (e.g., &ldquo;
   *        <code>_files</code>&rdquo;.) If <code>null</code> then no folder is
   *        created.
   * @return A reservation object that is passed to
   *         {@link #then(Reservation, File, File)}.
   * @throws Exception
   */
  Reservation reserve( Article article, String file_suffix, String folder_suffix )
    throws Exception;

  /**
   * Reserve a unique file and/or folder for saving the processed document.
   * Sometimes we need to know the final name of the file and/or folder to be
   * produced before we actually commence processing the article.
   * 
   * @param base_name
   *        A prefix to use in creating the unique names of the file and/or
   *        folder.
   * @param file_suffix
   *        The suffix to add to the name of the file (e.g., &ldquo;
   *        <code>.txt</code>&rdquo;.) If <code>null</code> then no file is
   *        created.
   * @param folder_suffix
   *        The suffix to add to the name of the folder (e.g., &ldquo;
   *        <code>_files</code>&rdquo;.) If <code>null</code> then no folder is
   *        created.
   * @return A reservation object that is passed to
   *         {@link #then(Reservation, File, File)}.
   * @throws Exception
   */
  Reservation reserve( String base_name, String file_suffix,
      String folder_suffix ) throws Exception;

  /**
   * Perform the post-processing action on the document.
   * 
   * @param reservation
   *        The reservation object produces by a call to <code>reserve</code>.
   * @param from_file
   *        The file to be copied to the reserve location. (<code>null</code> if
   *        there is no file to copy.)
   * @param from_folder
   *        The folder who's contents should be copied to the reserve location.
   *        (<code>null</code> if there is no folder to copy.)
   * @throws Exception
   */
  void then( Reservation reservation, File from_file, File from_folder )
    throws Exception;
} public final class ThenFactory
{
  /**
   * Create the specified {@link Then} handler.
   * 
   * @param prefix
   *        The prefix of the keys to read in the configuration.
   * @param config
   *        The configuration to read.
   * @return The handler object.
   */
  public static Then get( final String prefix, final Config config )
  {
    final String str = config.getString(prefix + "then");

    if ( Str.equalsIgnoreCase(str, "calibre") )
      return new CalibreThen(prefix, config);
    if ( Str.equalsIgnoreCase(str, "save") )
      return new SaveThen(prefix, config);

    if ( Str.isEmpty(str) )
      throw new IllegalStateException(prefix + "then action not specified.");
    throw new IllegalStateException("Unknown " + prefix + "then action: " + str);
  }

  private ThenFactory()
  {
    // empty
  }
}
import java.io.File;
import java.io.IOException;
import org.jsoup.nodes.Document;
import com.evanmclean.erudite.Article;
import com.evanmclean.erudite.Erudite;
import com.evanmclean.erudite.HNSearch;
import com.evanmclean.erudite.ImageHandlerFactory;
import com.evanmclean.erudite.Processor;
import com.evanmclean.erudite.Source;
import com.evanmclean.erudite.Template;
import com.evanmclean.erudite.TemplateFactory;
import com.evanmclean.erudite.config.Config;
import com.evanmclean.erudite.config.ConfigUtils;
import com.evanmclean.erudite.misc.Doc;
import com.evanmclean.erudite.misc.EpubWrapHack;
import com.evanmclean.erudite.misc.Utils;
import com.evanmclean.erudite.then.Reservation;
import com.evanmclean.erudite.then.Then;
import com.evanmclean.erudite.then.ThenFactory;
import com.evanmclean.evlib.lang.Str;
package com.evanmclean.erudite.calibre;




/**
 * A processor for converting HTML documents to another format (such as epub or
 * mobi) and then adding them to a Calibre e&ndash;book library or saving them
 * in a folder. See the user documentation (in particular the sample
 * configuration file).
 * 
 * @author Evan M<sup>c</sup>Lean, <a href="http://evanmclean.com/"
 *         target="_blank">M<sup>c</sup>Lean Computer Services</a>
 */
public class EbookConvertProcessor implements Processor
{
  private final EbookConverter converter;
  private final Template template;
  private final String filetype;
  private final String fileSuffix;
  private final boolean doFootnotes;
  private final boolean doHNSearch;
  private final boolean doWrapHack;
  private final Then then;