public interface ImageService {

	/**
	 * Extracts image metadata from the specified URL. If it is unable to determine that the given URL contains an image
	 * this methods returns <strong>null</strong>.
	 * 
	 * @param imageUrl
	 * @return an Image object or null if no image found at the given URL.
	 * @throws IOException
	 *             if something goes wrong connecting to the image source.
	 * @throws HttpException
	 */
	Image getImageFromUrl(URL imageUrl) throws IOException;

	/**
	 * Extracts image metadata from the specified resource. If it is unable to determine that the given URL contains an
	 * image this methods returns <strong>null</strong>.
	 * 
	 * @param resource
	 * @return an Image object or null if no image found at the given URL.
	 * @throws IOException
	 *             if something goes wrong connecting to the image source.
	 * @throws HttpException
	 */
	Image getImageFromResource(ReadableResource resource) throws IOException;

} public interface ReadableResource {

	URL getUrl();

	/**
	 * Move the current URL to the redirection path and set the specified url as the actual resource url.
	 * 
	 * @param url
	 *            the redirectionUrl
	 */
	void makeRedirection(URL url);

	Optional<URL[]> getRedirectionPath();

	Optional<MediaType> getContentType();

	InputStream read() throws IOException;

	void discard();
} @Service
public interface Scraper {

	/**
	 * Extract the list of structured contents from the given URL.
	 * 
	 * @param url
	 *            of the web resource
	 * @return the {@link WebContent} object describing the contents of the URL and all alias URLs. Or null if no
	 *         content is found.
	 * @throws IOException
	 *             is an IO error occurs while connecting to the URL.
	 * @throws HttpException
	 */
	WebContent scrapContent(String url) throws IOException;

	/**
	 * Extract the list of structured contents from the given URL.
	 * 
	 * @param url
	 *            of the web resource
	 * @return the {@link WebContent} object describing the contents of the URL and all alias URLs. Or null if no
	 *         content is found.
	 * @throws IOException
	 *             is an IO error occurs while connecting to the URL.
	 * @throws HttpException
	 */
	<E extends WebContent> E scrapContent(String url, Class<E> contentType) throws IOException;

	/**
	 * Extract the list of structured contents from the given URL.
	 * 
	 * @param url
	 *            of the web resource
	 * @return the {@link WebContent} object describing the contents of the URL and all alias URLs. Or null if no
	 *         content is found.
	 * @throws IOException
	 *             is an IO error occurs while connecting to the URL.
	 * @throws HttpException
	 */
	WebContent scrapContent(ReadableResource resource) throws IOException;

	/**
	 * Extract the list of structured contents from the given URL.
	 * 
	 * @param url
	 *            of the web resource
	 * @return the {@link WebContent} object describing the contents of the URL and all alias URLs. Or null if no
	 *         content is found.
	 * @throws IOException
	 *             is an IO error occurs while connecting to the URL.
	 * @throws HttpException
	 */
	<E extends WebContent> E scrapContent(ReadableResource resource, Class<E> contentType) throws IOException;

} public interface ServiceLocator {

	Scraper getScraper();

	HttpService getHttpService();

	ImageService getImageService();
} public class Feed extends WebContent {

	private String title;

	public Feed(String url, String... aliasUrls) {
		super(url, aliasUrls);
	}

	@Override
	public Type getType() {
		return Type.FEED;
	}

	/**
	 * @return the title of this feed
	 */
	public String getTitle() {
		return title;
	}

	/**
	 * @param title
	 *            the title to set
	 */
	public void setTitle(String title) {
		this.title = title;
	}

	@Override
	public boolean equals(Object obj) {
		if (!super.equals(obj) || !(obj instanceof Feed)) {
			return false;
		}
		Feed that = (Feed) obj;
		return Objects.equal(this.title, that.title);
	}

	@Override
	public String toString() {
		return toStringHelper(this).add("title", title).toString();
	}
} public class Image extends WebContent {

	private Integer width, height;

	public Image(String url, String... aliasUrls) {
		super(url, aliasUrls);
	}

	/**
	 * @return the width in pixels
	 */
	public Integer getWidth() {
		return width;
	}

	/**
	 * @param width
	 *            the width in pixels to set
	 */
	public void setWidth(Integer width) {
		this.width = width;
	}

	/**
	 * @return the height in pixels
	 */
	public Integer getHeight() {
		return height;
	}

	/**
	 * @param height
	 *            the height in pixels to set
	 */
	public void setHeight(Integer height) {
		this.height = height;
	}

	@Override
	public Type getType() {
		return Type.IMAGE;
	}

	@Override
	public String toString() {
		return toStringHelper(this).add("with", width).add("height", height).toString();
	}
} public class Video extends WebContent {

	private String title;
	private String hostingService;
	private String description;
	private String thumbnailUrl;
	private Integer duration;
	private Double averageRating;
	private String[] categories;
	private String htmlUrl;

	public Video(String url, String... aliasUrls) {
		super(url, aliasUrls);
	}

	public String getTitle() {
		return title;
	}

	public void setTitle(String title) {
		this.title = title;
	}

	public String getDescription() {
		return description;
	}

	public void setDescription(String description) {
		this.description = description;
	}

	public String getThumbnailUrl() {
		return thumbnailUrl;
	}

	public void setThumbnailUrl(String thumbnailUrl) {
		this.thumbnailUrl = thumbnailUrl;
	}

	public Double getAverageRating() {
		return averageRating;
	}

	public void setAverageRating(Double averageRating) {
		this.averageRating = averageRating;
	}

	public void setAverageRating(Float averageRating) {
		this.averageRating = averageRating != null ? averageRating.doubleValue() : null;
	}

	public String[] getCategories() {
		return categories;
	}

	public void setCategories(String[] categories) {
		this.categories = categories;
	}

	public void setCategories(Collection<String> categories) {
		this.categories = categories != null ? categories.toArray(new String[categories.size()]) : null;
	}

	public Integer getDuration() {
		return duration;
	}

	public void setDuration(Integer duration) {
		this.duration = duration;
	}

	public String getHostingService() {
		return hostingService;
	}

	public void setHostingService(String hostingService) {
		this.hostingService = hostingService;
	}

	public void setHtmlUrl(String htmlUrl) {
		this.htmlUrl = htmlUrl;
	}

	public String getHtmlUrl() {
		return htmlUrl;
	}

	@Override
	public Type getType() {
		return Type.VIDEO;
	}

	@Override
	public boolean equals(Object obj) {
		if (!super.equals(obj) || !(obj instanceof Video)) {
			return false;
		}
		Video that = (Video) obj;
		return Objects.equal(this.title, that.title) && Objects.equal(this.description, that.description)
				&& Objects.equal(this.thumbnailUrl, that.thumbnailUrl)
				&& Objects.equal(this.averageRating, that.averageRating)
				&& Arrays.equals(this.categories, that.categories) && Objects.equal(this.duration, that.duration)
				&& Objects.equal(this.hostingService, that.hostingService) && Objects.equal(this.htmlUrl, that.htmlUrl);
	}

	@Override
	public String toString() {
		return toStringHelper(this).add("title", title).add("description", description)
				.add("thumbnailUrl", thumbnailUrl).add("averageRating", averageRating)
				.add("categories", Arrays.toString(categories)).add("duration", duration)
				.add("hostingService", hostingService).add("htmlUrl", htmlUrl).toString();
	}
} public abstract class WebContent {

	public static enum Type {
		WEBPAGE,
		IMAGE,
		VIDEO,
		AUDIO,
		FEED
	}

	private final String url;
	private final String[] aliasUrls;
	private Set<String> keywords;

	WebContent(String url, String... aliasUrls) {
		this.url = url;
		this.aliasUrls = aliasUrls;
	}

	/**
	 * Gets the preferred URL of this content
	 * 
	 * @return the url
	 */
	public String getUrl() {
		return url;
	}

	/**
	 * Gets the known alias URLs where this content can be also located. The order is defined by the redirection path.
	 * 
	 * @return the referingUrls
	 */
	public String[] getAliasUrls() {
		return aliasUrls;
	}

	/**
	 * @return the type of this web content.
	 */
	public abstract Type getType();

	/**
	 * @return the keywords of this web content.
	 */
	public Set<String> getKeywords() {
		return keywords;
	}

	/**
	 * @param keywords
	 *            the keywords to set
	 */
	public void setKeywords(Set<String> keywords) {
		this.keywords = keywords;
	}

	@Override
	public int hashCode() {
		return Objects.hashCode(url, getType());
	}

	@Override
	public boolean equals(Object obj) {
		if (obj == null || !(obj instanceof WebContent)) {
			return false;
		}
		WebContent that = (WebContent) obj;
		return Objects.equal(this.url, that.url) && Objects.equal(this.getType(), that.getType())
				&& Arrays.equals(this.aliasUrls, that.aliasUrls) && Objects.equal(this.keywords, that.keywords);
	}

	protected ToStringHelper toStringHelper(Object instance) {
		return Objects.toStringHelper(this).omitNullValues().add("url", url)
				.add("aliasUrls", Arrays.toString(aliasUrls)).add("keywords", keywords);
	}

} public class Webpage extends WebContent {

	private String title;
	private String description;
	private Image previewImage;
	private List<Feed> feeds;
	private List<WebContent> embeddedContent;

	public Webpage(String url, String... aliasUrls) {
		super(url, aliasUrls);
	}

	/**
	 * @return the title defined in the HEAD section of this webpage
	 */
	public String getTitle() {
		return title;
	}

	/**
	 * @param title
	 *            the page title to set
	 */
	public void setTitle(String title) {
		this.title = title;
	}

	/**
	 * @return the description of this webpage as it appears in its meta section.
	 */
	public String getDescription() {
		return description;
	}

	/**
	 * @param description
	 *            the description to set.
	 */
	public void setDescription(String description) {
		this.description = description;
	}

	/**
	 * @return the preview image defined in the metadata of this webpage.
	 */
	public Image getPreviewImage() {
		return previewImage;
	}

	/**
	 * 
	 * @param previewImage
	 *            the preview image to set
	 */
	public void setPreviewImage(Image previewImage) {
		this.previewImage = previewImage;
	}

	/**
	 * @return the feeds (RSS/Atom) that are linked in the metadata of this webpage.
	 */
	public List<Feed> getFeeds() {
		return feeds;
	}

	/**
	 * @param feeds
	 *            the feeds to set
	 */
	public void setFeeds(List<Feed> feeds) {
		this.feeds = feeds;
	}

	public void addFeed(Feed feed) {
		if (feeds == null) {
			feeds = new ArrayList<Feed>();
		}
		feeds.add(feed);
	}

	public List<WebContent> getEmbeddedContent() {
		return embeddedContent;
	}

	public void setEmbeddedContent(List<WebContent> embeddedContent) {
		this.embeddedContent = embeddedContent;
	}

	public void addEmbeddedContent(WebContent content) {
		if (embeddedContent == null) {
			embeddedContent = new ArrayList<WebContent>();
		}
		embeddedContent.add(content);
	}

	@Override
	public Type getType() {
		return Type.WEBPAGE;
	}

	@Override
	public String toString() {
		return toStringHelper(this).add("title", title).add("description", description)
				.add("previewImage", previewImage).add("feeds", feeds).add("embeddedContent", embeddedContent)
				.toString();
	}

} public class URLUtil {

	public static String asString(URL url) {
		return url.toString();
	}

	public static String[] asString(URL[] urls) {
		if (urls == null) {
			return null;
		}
		return asString(Arrays.asList(urls)).toArray(new String[] {});
	}

	public static Collection<String> asString(Collection<URL> urls) {
		if (urls == null) {
			return null;
		}
		Collection<String> ret = Collections2.transform(urls, new Function<URL, String>() {
			@Override
			public String apply(URL input) {
				return asString(input);
			}
		});
		return ret;
	}

	public static URL getAbsoluteUrl(URL baseUrl, String relativeUrl) throws MalformedURLException {
		if (isAbsolute(relativeUrl)) {
			return new URL(relativeUrl);
		}
		return new URL(baseUrl, relativeUrl);
	}

	public static boolean isAbsolute(String url) {
		return url.matches("^.*://.*");
	}

}
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.tika.parser.html.HtmlEncodingDetector;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Optional;
import com.google.common.net.MediaType;
import com.lumata.lib.lupa.ImageService;
import com.lumata.lib.lupa.ReadableResource;
import com.lumata.lib.lupa.Scraper;
import com.lumata.lib.lupa.ServiceLocator;
import com.lumata.lib.lupa.content.Feed;
import com.lumata.lib.lupa.content.Image;
import com.lumata.lib.lupa.content.Video;
import com.lumata.lib.lupa.content.WebContent;
import com.lumata.lib.lupa.content.Webpage;
import com.lumata.lib.lupa.util.URLUtil;

package com.lumata.lib.lupa.extractor.internal;




public class HtmlExtractor extends AbstractExtractor<WebContent> {

	private static final Logger LOG = LoggerFactory.getLogger(HtmlExtractor.class);

	private static final String CONFIG_CONTENT_SELECTOR = "contentSelector";
	private static final String CONFIG_CONTENT_SELECTOR_DEFAULT = "article, div[id~=.*article.*ontent.*], div[class~=.*media-instance-container.*],div[class~=.*article.*ontent.*], div[id~=.*article.*], div[class~=.*article.*], div[id~=.*post.*], div[class~=.*post.*], div[class~=.*Contenido.*], div[id~=.*content.*], div[class~=.*content.*], div[id~=.*main.*], div[class~=.*main.*], div[id~=.*divsinglecolumnminwidth.*]";

	private static final String CONFIG_SCRAP_BODY = "scrapBody";
	private static final Boolean CONFIG_SCRAP_BODY_DEFAULT = true;

	private static final String CONFIG_EXCLUDED_ELEMENTS = "excludedElements";
	private static final String CONFIG_EXCLUDED_ELEMENTS_DEFAULT = "div[id~=.*nav.*], div[id~=.*bar.*], div[id~=.*share.*], div[class~=.*nav.*], div[class~=.*share.*], div[class~=.*tags.*], *[style~=.*display:none.*],div[id~=ad-area]";

	private static final Pattern REFRESH_URL_PATTERN = Pattern.compile(".*URL=(.*)", Pattern.CASE_INSENSITIVE);
	private static final int BUFFER_2KB = 2048;
	private static final String UTF8 = "utf8";

	@Override
	public WebContent extractContent(ReadableResource resource, ServiceLocator serviceLocator) throws IOException {
		try {
			Document doc = parseWithDeclaredCharset(resource);
			return scrapDocument(doc, resource, serviceLocator.getScraper(), serviceLocator.getImageService());
		} catch (RedirectDetected redirection) {
			LOG.debug("An HTML declared redirection has been detected from {} to {}", resource.getUrl(),
					redirection.redirectUrl);
			resource.makeRedirection(new URL(redirection.redirectUrl));
			return serviceLocator.getScraper().scrapContent(resource);
		}
	}

	@Override
	public Class<WebContent> getExtractableWebContentType() {
		return WebContent.class;
	}

	/* -----helper methods ---------------------------------- */
	private Document parseWithDeclaredCharset(ReadableResource resource) throws RedirectDetected, IOException {

		// 1) Create a buffered stream to be able to read ahead
		InputStream stream = new BufferedInputStream(resource.read());
		stream.mark(BUFFER_2KB);

		// 2) detect encoding from HTTP headers or use default (UTF-8)
		Charset httpCharset = getResourceCharsetOrDefaultCharset(resource);
		LOG.debug("Initial charset (http header or default) for {} is {}", resource.getUrl(), httpCharset);

		// 3) Read head section ahead
		String headText = readHead(stream, httpCharset.name(), BUFFER_2KB);
		stream.reset();

		// 4) Look for encodings declared on the HTML and use this one if found
		Charset effectiveCharset = getHtmlDeclaredCharset(headText).or(httpCharset);
		LOG.debug("Final charset (after HTML charset extraction) used for parsing {} is {}", resource.getUrl(),
				effectiveCharset);
		String encoding = effectiveCharset != null ? effectiveCharset.name() : httpCharset.name();

		// 5) Detect any possible redirects declared on the HTML
		detectHtmlDeclaredRedirects(headText, encoding, resource.getUrl());

		// 6) Parse the document
		Document doc = Jsoup.parse(stream, encoding, resource.getUrl().toString());

		return doc;

	}

	private Charset getResourceCharsetOrDefaultCharset(ReadableResource resource) {
		return resource.getContentType().or(MediaType.HTML_UTF_8).charset().or(Charset.forName(UTF8));
	}

	private Optional<Charset> getHtmlDeclaredCharset(String head) throws IOException {
		HtmlEncodingDetector encodingDetector = new HtmlEncodingDetector();
		return Optional.fromNullable(encodingDetector.detect(new ByteArrayInputStream(head.getBytes()), null));
	}

	private void detectHtmlDeclaredRedirects(String headText, String encoding, URL originalUrl) throws RedirectDetected {
		Element head = Jsoup.parse(headText, encoding);
		// if refresh tag is present do redirect
		Element refresh = head.select("meta[http-equiv=refresh]").first();
		if (refresh != null) {
			String content = refresh.attr("content");
			if (content != null) {
				Matcher matcher = REFRESH_URL_PATTERN.matcher(content);
				if (matcher.matches()) {
					String redirectUrl = matcher.group(1);
					try {
						redirectUrl = URLUtil.getAbsoluteUrl(originalUrl, redirectUrl).toString();
						// only redirect if URL is not the same to avoid infinite loops
						if (!originalUrl.toString().equals(redirectUrl)) {
							throw new RedirectDetected(redirectUrl);
						}
					} catch (MalformedURLException e) {
						LOG.warn("Redirect URL ({}) found in {} but is invalid", redirectUrl, originalUrl);
					}
				}
			}
		}

	}


	private WebContent scrapDocument(Document doc, ReadableResource resource, Scraper scraper, ImageService imageService) {
