@SuppressWarnings({ "serial" })
public class AbortedFetchException extends BaseFetchException {
    private AbortedFetchReason _abortReason;

    public AbortedFetchException() {
        super();
    }

    public AbortedFetchException(String url, AbortedFetchReason abortReason) {
        super(url, "Aborted due to " + abortReason);

        _abortReason = abortReason;
    }

    public AbortedFetchException(String url, String msg, AbortedFetchReason abortReason) {
        super(url, msg);

        _abortReason = abortReason;
    }

    public AbortedFetchReason getAbortReason() {
        return _abortReason;
    }
} public enum AbortedFetchReason {
    // WARNING - adding new reasons requires changes to
    // AbortedFetchException.mapToUrlStatus

    SLOW_RESPONSE_RATE, // Response rate back from server was below minimum.
    INVALID_MIMETYPE, // FetcherPolicy doesn't specify this as a valid mime-type
    INTERRUPTED, // Fetch was interrupted (typically by FetchBuffer calling
                 // executor.terminate())
    CONTENT_SIZE, // Content exceeds Fetcher.getMaxContentSize()
    FETCH_DURATION_EXCEEDED, // Fetch has exceeded the duration of time it was allotted 
} @SuppressWarnings("serial")
public abstract class BaseFetcher implements Serializable {

    public static final int DEFAULT_MAX_CONTENT_SIZE = 64 * 1024;

    protected Map<String, Integer> _maxContentSizes = new HashMap<>();
    protected int _defaultMaxContentSize = DEFAULT_MAX_CONTENT_SIZE;
    protected Set<String> _validMimeTypes = new HashSet<>();

    public BaseFetcher() {
    }

    public void setDefaultMaxContentSize(int defaultMaxContentSize) {
        _defaultMaxContentSize = defaultMaxContentSize;
    }

    public int getDefaultMaxContentSize() {
        return _defaultMaxContentSize;
    }

    public void setMaxContentSize(String mimeType, int maxContentSize) {
        _maxContentSizes.put(mimeType, maxContentSize);
    }

    public int getMaxContentSize(String mimeType) {
        Integer result = _maxContentSizes.get(mimeType);
        if (result == null) {
            result = getDefaultMaxContentSize();
        }

        return result;
    }

    public Set<String> getValidMimeTypes() {
        return _validMimeTypes;
    }

    public void setValidMimeTypes(Set<String> validMimeTypes) {
        _validMimeTypes = new HashSet<String>(validMimeTypes);
    }

    public void addValidMimeTypes(Set<String> validMimeTypes) {
        _validMimeTypes.addAll(validMimeTypes);
    }

    public void addValidMimeType(String validMimeType) {
        _validMimeTypes.add(validMimeType);
    }

    public FetchedResult get(String url) throws BaseFetchException {
        return get(url, null);
    }

    protected static String getMimeTypeFromContentType(String contentType) {
        String result = "";
        MediaType mt = MediaType.parse(contentType);
        if (mt != null) {
            result = mt.getType() + "/" + mt.getSubtype();
        }

        return result;
    }

    /**
     * Get the content stored in the resource referenced by the 'url' parameter.
     * 
     * @param url a string url for which to get content
     * @param payload a populated {@link crawlercommons.fetcher.Payload}
     * @return the {@link crawlercommons.fetcher.FetchedResult} associated with the URL
     * @throws BaseFetchException if an error results from fetching the url.
     */
    public abstract FetchedResult get(String url, Payload payload) throws BaseFetchException;

    /**
     * Terminate any async request being processed.
     * 
     */
    public abstract void abort();

} public class FetchedResult {
    private final String _baseUrl;
    private final String _fetchedUrl;
    private final long _fetchTime;
    private final byte[] _content;
    private final String _contentType;
    private final int _responseRate;
    private final Headers _headers;
    private final String _newBaseUrl;
    private final int _numRedirects;
    private final String _hostAddress;
    private final int _statusCode; // HTTP status code
    private final String _reasonPhrase; // HTTP reason phrase, or null
    private Payload _payload;

    public FetchedResult(String baseUrl, String redirectedUrl, long fetchTime, Headers headers, byte[] content, String contentType, int responseRate, Payload payload, String newBaseUrl,
                    int numRedirects, String hostAddress, int statusCode, String reasonPhrase) {
        _payload = payload;

        if (baseUrl == null) {
            throw new InvalidParameterException("baseUrl cannot be null");
        }

        if (redirectedUrl == null) {
            throw new InvalidParameterException("redirectedUrl cannot be null");
        }

        if (headers == null) {
            throw new InvalidParameterException("headers cannot be null");
        }

        if (content == null) {
            throw new InvalidParameterException("content cannot be null");
        }

        if (contentType == null) {
            throw new InvalidParameterException("contentType cannot be null");
        }

        if (hostAddress == null) {
            throw new InvalidParameterException("hostAddress cannot be null");
        }

        _baseUrl = baseUrl;
        _fetchedUrl = redirectedUrl;
        _fetchTime = fetchTime;
        _content = content;
        _contentType = contentType;
        _responseRate = responseRate;
        _headers = headers;
        _newBaseUrl = newBaseUrl;
        _numRedirects = numRedirects;
        _hostAddress = hostAddress;
        _statusCode = statusCode;
        _reasonPhrase = reasonPhrase;
    }

    public Payload getPayload() {
        return _payload;
    }

    public void setPayload(Payload payload) {
        _payload = payload;
    }

    public String getBaseUrl() {
        return _baseUrl;
    }

    public String getFetchedUrl() {
        return _fetchedUrl;
    }

    public long getFetchTime() {
        return _fetchTime;
    }

    public byte[] getContent() {
        return _content;
    }

    public int getContentLength() {
        return _content.length;
    }

    public String getContentType() {
        return _contentType;
    }

    public int getResponseRate() {
        return _responseRate;
    }

    public Headers getHeaders() {
        return _headers;
    }

    public String getNewBaseUrl() {
        return _newBaseUrl;
    }

    public int getNumRedirects() {
        return _numRedirects;
    }

    public String getHostAddress() {
        return _hostAddress;
    }

    public int getStatusCode() {
        return _statusCode;
    }

    public String getReasonPhrase() {
        return _reasonPhrase;
    }

    /**
     * Produces a neat report containing everything from a {@link FetchedResult}
     * . The order of the report is based on the logical population of
     * FetchedReport entities as per a non-public algorithm within
     * {@link crawlercommons.fetcher.http.SimpleHttpFetcher}.
     * 
     * @return returns a String report of the FetchedResult.
     */
    public String report() {
        StringBuilder report = new StringBuilder();
        report.append("FetchedResult Report:\n");
        report.append("*********************\n");
        report.append("    BaseUrl       : " + getBaseUrl() + "\n");
        report.append("    Headers       : __\n");
        for (String header : getHeaders().names()) {
            String mdString = header + "=" + getHeaders().getValues(header);
            report.append("                   " + mdString + "\n");
        }
        report.append("    StatusCode    : " + getStatusCode() + "\n");
        report.append("    ReasonPhrase  : " + getReasonPhrase() + "\n");
        report.append("    NumRedirects  : " + getNumRedirects() + "\n");
        report.append("    NewBaseUrl    : " + getNewBaseUrl() + "\n");
        report.append("    HostAddress   : " + getHostAddress() + "\n");
        report.append("    ResponseRate  : " + getResponseRate() + "\n");
        report.append("    PayLoad       : __\n");
        for (String payLoad : getPayload().keySet()) {
            String payString = payLoad + getPayload().get(payLoad);
            report.append("                   " + payString + "\n");
        }
        report.append("    FetchTime     : " + getFetchTime() + "\n");
        report.append("    FetchedUrl    : " + getFetchedUrl() + "\n");
        report.append("    ContentType   : " + getContentType() + "\n");
        report.append("    ContentLength : " + getContentLength() + "\n");
        report.append("    Content       : " + new String(getContent(), Charset.defaultCharset()) + "\n");
        report.append("*********************\n");
        report.append("End of Report:\n");
        return report.toString();

    }
} @SuppressWarnings({ "serial" })
public class IOFetchException extends BaseFetchException {

    public IOFetchException() {
        super();
    }

    public IOFetchException(String url, IOException e) {
        super(url, e);
    }

} public class Payload implements Map<String, Object> {

    private Map<String, Object> _data;

    public Payload() {
        _data = new HashMap<String, Object>();
    }

    public void clear() {
        _data.clear();
    }

    public boolean containsKey(Object key) {
        return _data.containsKey(key);
    }

    public boolean containsValue(Object value) {
        return _data.containsValue(value);
    }

    public Set<java.util.Map.Entry<String, Object>> entrySet() {
        return _data.entrySet();
    }

    public boolean equals(Object o) {
        return _data.equals(o);
    }

    public Object get(Object key) {
        return _data.get(key);
    }

    public int hashCode() {
        return _data.hashCode();
    }

    public boolean isEmpty() {
        return _data.isEmpty();
    }

    public Set<String> keySet() {
        return _data.keySet();
    }

    public Object put(String key, Object value) {
        return _data.put(key, value);
    }

    public void putAll(Map<? extends String, ? extends Object> m) {
        _data.putAll(m);
    }

    public Object remove(Object key) {
        return _data.remove(key);
    }

    public int size() {
        return _data.size();
    }

    public Collection<Object> values() {
        return _data.values();
    }

} @SuppressWarnings({ "serial" })
public class RedirectFetchException extends BaseFetchException {

    // Possible redirect exception types.

    public enum RedirectExceptionReason {
        TOO_MANY_REDIRECTS, // - Request for original URL tried too many hops.
        PERM_REDIRECT_DISALLOWED, // - RedirectMode disallows a permanent
                                  // redirect.
        TEMP_REDIRECT_DISALLOWED, // - RedirectMode disallows a temp redirect.
        SEE_OTHER_DISALLOWED // - to handle 303 redirects, usually after POST
                             // method & RESTful applications
    }

    private String _redirectedUrl;
    private RedirectExceptionReason _reason;

    public RedirectFetchException() {
        super();
    }

    public RedirectFetchException(String url, String redirectedUrl, RedirectExceptionReason reason) {
        super(url, "Too many redirects");
        _redirectedUrl = redirectedUrl;
        _reason = reason;
    }

    public String getRedirectedUrl() {
        return _redirectedUrl;
    }

    public RedirectExceptionReason getReason() {
        return _reason;
    }

} public enum RedirectExceptionReason {
    TOO_MANY_REDIRECTS, // - Request for original URL tried too many hops.
    PERM_REDIRECT_DISALLOWED, // - RedirectMode disallows a permanent
                              // redirect.
    TEMP_REDIRECT_DISALLOWED, // - RedirectMode disallows a temp redirect.
    SEE_OTHER_DISALLOWED // - to handle 303 redirects, usually after POST
                         // method & RESTful applications
} public enum RedirectMode {
    FOLLOW_ALL, // Fetcher will try to follow all redirects
    FOLLOW_TEMP, // Temp redirects are automatically followed, but not
                 // permanent.
    FOLLOW_NONE // No redirects are followed.
} public class FixedStatusResponseHandler extends AbstractHandler {
    
    private int _status;

    public FixedStatusResponseHandler(int status) {
        _status = status;
    }

    @Override
    public void handle(String pathInContext, Request baseRequest,
                       HttpServletRequest request, HttpServletResponse response) throws IOException {
        response.sendError(_status, "Pre-defined error fetching: " + pathInContext);
    }
} public class RandomResponseHandler extends AbstractHandler {

    private static final long DEFAULT_DURATION = 1000L;

    private int _length;
    private long _duration;
    private Random _rand;

    /**
     * Create an HTTP response handler that sends random data back at a
     * particular rate.
     * 
     * @param length
     *            - number of bytes to return
     * @param duration
     *            - duration for response, in milliseconds.
     */
    public RandomResponseHandler(int length, long duration) {
        _length = length;
        _duration = duration;
        _rand = new Random();
    }

    /**
     * Send back <length> bytes of random data in one second.
     * 
     * @param length
     *            - number of bytes to return
     */
    public RandomResponseHandler(int length) {
        this(length, DEFAULT_DURATION);
    }

    @Override
    public void handle(String pathInContext, Request baseRequest,
                       HttpServletRequest request, HttpServletResponse response) throws IOException {
        response.setContentLength(_length);
        response.setContentType("text/html");
        response.setStatus(200);

        OutputStream os = response.getOutputStream();
        long startTime = System.currentTimeMillis();
        byte[] bytes = new byte[1];

        try {
            for (long i = 0; i < _length; i++) {
                _rand.nextBytes(bytes);
                os.write(bytes[0]);

                // Given i/_length as % of data written, we know that
                // this * duration is the target elapsed time. Figure out
                // how much to delay to make it so.
                long curTime = System.currentTimeMillis();
                long elapsedTime = curTime - startTime;
                long targetTime = (i * _duration) / _length;
                if (elapsedTime < targetTime) {
                    Thread.sleep(targetTime - elapsedTime);
                }
            }
        } catch (InterruptedException e) {
            response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Response handler interrupted");
        }
    }
} public class ResourcesResponseHandler extends AbstractHandler {
    private String _testContext = "";

    /**
     * Create an HTTP response handler that sends data back from files on the
     * classpath TODO KKr - use regular Jetty support for this, via setting up
     * HttpContext
     * 
     */
    public ResourcesResponseHandler() {
    }

    public ResourcesResponseHandler(String testContext) {
        _testContext = testContext;
    }

    @Override
    public void handle(String pathInContext, Request baseRequest,
                       HttpServletRequest request, HttpServletResponse response) throws IOException {
        // Get the resource.
        URL path = ResourcesResponseHandler.class.getResource(_testContext + pathInContext);
        if (path == null) {
            response.sendError(HttpServletResponse.SC_NOT_FOUND, "Resource not found: " + pathInContext);
            return;
        }

        try {
            File file = new File(path.getFile());
            byte[] bytes = new byte[(int) file.length()];
            @SuppressWarnings("resource")
            DataInputStream in = new DataInputStream(new FileInputStream(file));
            in.readFully(bytes);

            response.setContentLength(bytes.length);
            if (file.getName().endsWith(".png")) {
                response.setContentType("image/png");
            } else {
                response.setContentType("text/html");
            }
            response.setStatus(200);

            OutputStream os = response.getOutputStream();
            os.write(bytes);
        } catch (Exception e) {
            response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
        }
    }
} public class SimulationWebServer {

    private Server _server;

    public Server startServer(Handler handler, int port) throws Exception {
        _server = new Server(port);
        _server.setHandler(handler);
        _server.start();
        return _server;
    }

    public void stopServer() throws Exception {
        if (_server != null) {
            _server.stop();
        }
    }

    public Server getServer() {
        return _server;
    }

} public class TestUtils {

    // User agent for when we're not doing external fetching, so we just need a
    // fake name.
    public static final UserAgent CC_TEST_AGENT = new UserAgent("test", "test@domain.com", "http://test.domain.com");

}
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.ConnectException;
import java.nio.charset.Charset;
import java.util.HashSet;
import java.util.Set;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import crawlercommons.fetcher.AbortedFetchException;
import crawlercommons.fetcher.AbortedFetchReason;
import crawlercommons.fetcher.BaseFetcher;
import crawlercommons.fetcher.FetchedResult;
import crawlercommons.fetcher.IOFetchException;
import crawlercommons.fetcher.Payload;
import crawlercommons.fetcher.RedirectFetchException;
import crawlercommons.fetcher.RedirectFetchException.RedirectExceptionReason;
import crawlercommons.fetcher.http.BaseHttpFetcher.RedirectMode;
import crawlercommons.test.FixedStatusResponseHandler;
import crawlercommons.test.RandomResponseHandler;
import crawlercommons.test.ResourcesResponseHandler;
import crawlercommons.test.SimulationWebServer;
import crawlercommons.test.TestUtils;

            response.setStatus(HttpStatus.SC_OK);
            response.setContentType("text/plain");

            response.setContentLength(content.length());
            response.getOutputStream().write(content.getBytes("UTF-8"));
        }
    }

    private class MimeTypeResponseHandler extends AbstractHandler {

        private String _mimeType;

        public MimeTypeResponseHandler(String mimeType) {
            _mimeType = mimeType;
        }

        @Override
        public void handle(String pathInContext, Request baseRequest,
                           HttpServletRequest request, HttpServletResponse response) throws IOException {
            String content = "test";
            response.setStatus(HttpStatus.SC_OK);
            if (_mimeType != null) {
                response.setContentType(_mimeType);
            }

            response.setContentLength(content.length());
            response.getOutputStream().write(content.getBytes("UTF-8"));
        }
    }

    @Test
    public final void testConnectionTimeout() throws Exception {
        startServer(new ResourcesResponseHandler(), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        String url = "http://localhost:8088/simple-page.html";

        try {
            fetcher.get(url);
            fail("Exception not thrown");
        } catch (IOFetchException e) {
            assertTrue(e.getCause() instanceof ConnectException);
        }
    }

    @Test
    public final void testStaleConnection() throws Exception {
        startServer(new ResourcesResponseHandler(), 8089);
        ServerConnector sc = (ServerConnector) getServer().getConnectors()[0];
        sc.setSoLingerTime(-1);

        BaseFetcher fetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        String url = "http://localhost:8089/simple-page.html";
        fetcher.get(url);

        // TODO KKr - control keep-alive (linger?) value for Jetty, so we can
        // set it
        // to something short and thus make this sleep delay much shorter.
        Thread.sleep(2000);

        fetcher.get(url);
    }

    @Test
    public final void testFetchDurationTimeout() throws Exception {
        // Set up a response handler that provides a response for 2 seconds.
        startServer(new RandomResponseHandler(20000, 2 * 1000L), 8089);
        BaseHttpFetcher baseHttpFetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        baseHttpFetcher.setFetchDurationTimeoutInSeconds(1);  // Limit the fetch duration to 1 second.
        
        String url = "http://localhost:8089/test.html";
        try {
            baseHttpFetcher.get(url);
            fail("Exception not thrown");
        } catch (AbortedFetchException e) {
            assertTrue(e.getAbortReason() == AbortedFetchReason.FETCH_DURATION_EXCEEDED);
        }
    }
    
    @Test
    public final void testSlowServerTermination() throws Exception {
        // Need to read in more than 2 8K blocks currently, due to how
        // HttpClientFetcher
        // is designed...so use 20K bytes. And the duration is 2 seconds, so 10K
        // bytes/sec.
        startServer(new RandomResponseHandler(20000, 2 * 1000L), 8089);

        // Set up for a minimum response rate of 20000 bytes/second.
        BaseHttpFetcher fetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        fetcher.setMinResponseRate(20000);

        String url = "http://localhost:8089/test.html";
        try {
            fetcher.get(url);
            fail("Aborted fetch exception not thrown");
        } catch (AbortedFetchException e) {
            assertEquals(AbortedFetchReason.SLOW_RESPONSE_RATE, e.getAbortReason());
        }
    }

    @Test
    public final void testNotTerminatingSlowServers() throws Exception {
        // Return 1K bytes at 2K bytes/second - would normally trigger an
        // error.
        startServer(new RandomResponseHandler(1000, 500), 8089);

        // Set up for no minimum response rate.
        BaseHttpFetcher fetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        fetcher.setMinResponseRate(BaseHttpFetcher.NO_MIN_RESPONSE_RATE);

        String url = "http://localhost:8089/test.html";
        fetcher.get(url);
    }

    @Test
    public final void testLargeContent() throws Exception {
        BaseFetcher fetcher = new SimpleHttpFetcher(1, TestUtils.CC_TEST_AGENT);
        startServer(new RandomResponseHandler(fetcher.getDefaultMaxContentSize() * 2), 8089);

        String url = "http://localhost:8089/test.html";
        FetchedResult result = fetcher.get(url);