// Path: src/org/owasp/webscarab/model/Preferences.java
// public class Preferences {
//     
//     static Properties _props = new Properties();
//     private static String _location = null;
//     
//     /** Creates a new instance of Preferences */
//     private Preferences() {
//     }
//     
//     public static Properties getPreferences() {
//         return _props;
//     }
//     
//     public static void loadPreferences(String file) throws IOException {
//         // If we are given a filename to load, use it, otherwise
//         // look for a props file in the user's home directory
//         // if the file does not exist, use the standard defaults
//         
//         if (file == null) {
//             String sep = System.getProperty("file.separator");
//             String home = System.getProperty("user.home");
//             _location = home + sep + "WebScarab.properties";
//         } else {
//             _location = file;
//         }
//         
//         try {
//             Properties props = new Properties();
//             InputStream is = new FileInputStream(_location);
//             props.load(is);
//             _props = props;
//         } catch (FileNotFoundException fnfe) {
//             // we'll just use the defaults
//         }
//     }
//     
//     public static void savePreferences() throws IOException {
//         FileOutputStream fos = new FileOutputStream(_location);
//         _props.store(fos,"WebScarab Properties");
//         fos.close();
//     }
//     
//     public static void setPreference(String key, String value) {
//         _props.setProperty(key, value);
//     }
//     
//     public static String getPreference(String key) {
//         String result = System.getProperty(key);
//         if (result == null)
//             result = _props.getProperty(key);
//         return result;
//     }
//     
//     public static String getPreference(String key, String defaultValue) {
//         String result = getPreference(key);
//         if (result == null)
//             result = defaultValue;
//         return result;
//     }
//     
//     public static String remove(String key) {
//         return (String) _props.remove(key);
//     }
//     
// }


import org.owasp.webscarab.model.UrlModel;
import org.owasp.webscarab.model.Cookie;
import org.owasp.webscarab.model.NamedValue;
import org.owasp.webscarab.model.HttpUrl;
import org.owasp.webscarab.model.Preferences;
import org.owasp.webscarab.model.FrameworkModel;
import org.owasp.webscarab.model.FilteredUrlModel;
import org.owasp.webscarab.plugin.AbstractPluginModel;
import java.util.List;
import java.util.LinkedList;
import java.util.logging.Logger;
/*
 * SpiderModel.java
 *
 * Created on 04 March 2005, 03:11
 */

package org.owasp.webscarab.plugin.spider;




/**
 *
 * @author  rogan
 */
public class SpiderModel extends AbstractPluginModel {
    
    private FrameworkModel _model;
    private SpiderUrlModel _urlModel;
    
    private List<Link> _linkQueue = new LinkedList<Link>();
    
    private String _allowedDomains = null;
    private String _forbiddenPaths = null;
    private boolean _recursive = false;
    private boolean _cookieSync = false;
    
    private NamedValue[] _extraHeaders = null;
    
    private Logger _logger = Logger.getLogger(getClass().getName());
    
    /** Creates a new instance of SpiderModel */
    public SpiderModel(FrameworkModel model) {
        _model = model;
        _urlModel = new SpiderUrlModel(_model.getUrlModel());
        parseProperties();
    }
    
    public UrlModel getUrlModel() {
        return _urlModel;
    }
    
    public boolean isUnseen(HttpUrl url) {
        return _model.getUrlProperty(url, "METHODS") == null;
    }
    
    public boolean isForbidden(HttpUrl url) {
        if (_forbiddenPaths != null && !_forbiddenPaths.equals("")) {
            try {
                return url.toString().matches(getForbiddenPaths());
            } catch (Exception e) {
            }
        }
        return false;
    }
    
    public void addUnseenLink(HttpUrl url, HttpUrl referer) {
        if (url == null) {
            return;
        }
        if (isUnseen(url)) {
            String first = _model.getUrlProperty(url, "REFERER");
            if (first == null || first.equals("")) {
                _model.setUrlProperty(url, "REFERER", referer.toString());
            }
        }
    }
    
    public void queueLink(Link link) {
        // _logger.info("Queueing " + link);
        try {
            _model.readLock().acquire();
            _linkQueue.add(link);
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
        // _logger.info("Done queuing " + link);
    }
    
    public Link dequeueLink() {
        // _logger.info("Dequeueing a link");
        Link link = null;
        try {
            _model.readLock().acquire();
            if (_linkQueue.size() > 0) 
                link = _linkQueue.remove(0);
            if (_linkQueue.size() == 0) {
                setStatus("Idle");
            } else {
                setStatus(_linkQueue.size() + " links remaining");
            }
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
            // _logger.info("Done dequeuing a link " + link);
        }
        return link;
    }
    
    public void clearLinkQueue() {
        try {
            _model.readLock().acquire();
            _linkQueue.clear();
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
    }
    
    public int getQueuedLinkCount() {
        try {
            _model.readLock().acquire();
            return _linkQueue.size();
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
        return 0;
    }
    
    public Cookie[] getCookiesForUrl(HttpUrl url) {
        return _model.getCookiesForUrl(url);
    }
    
    public void addCookie(Cookie cookie) {
        _model.addCookie(cookie);
    }
    
    public void parseProperties() {
        String prop = "Spider.domains";
// Path: src/org/owasp/webscarab/model/Preferences.java
// public class Preferences {
//     
//     static Properties _props = new Properties();
//     private static String _location = null;
//     
//     /** Creates a new instance of Preferences */
//     private Preferences() {
//     }
//     
//     public static Properties getPreferences() {
//         return _props;
//     }
//     
//     public static void loadPreferences(String file) throws IOException {
//         // If we are given a filename to load, use it, otherwise
//         // look for a props file in the user's home directory
//         // if the file does not exist, use the standard defaults
//         
//         if (file == null) {
//             String sep = System.getProperty("file.separator");
//             String home = System.getProperty("user.home");
//             _location = home + sep + "WebScarab.properties";
//         } else {
//             _location = file;
//         }
//         
//         try {
//             Properties props = new Properties();
//             InputStream is = new FileInputStream(_location);
//             props.load(is);
//             _props = props;
//         } catch (FileNotFoundException fnfe) {
//             // we'll just use the defaults
//         }
//     }
//     
//     public static void savePreferences() throws IOException {
//         FileOutputStream fos = new FileOutputStream(_location);
//         _props.store(fos,"WebScarab Properties");
//         fos.close();
//     }
//     
//     public static void setPreference(String key, String value) {
//         _props.setProperty(key, value);
//     }
//     
//     public static String getPreference(String key) {
//         String result = System.getProperty(key);
//         if (result == null)
//             result = _props.getProperty(key);
//         return result;
//     }
//     
//     public static String getPreference(String key, String defaultValue) {
//         String result = getPreference(key);
//         if (result == null)
//             result = defaultValue;
//         return result;
//     }
//     
//     public static String remove(String key) {
//         return (String) _props.remove(key);
//     }
//     
// }


// Path: src/org/owasp/webscarab/plugin/spider/SpiderModel.java
import org.owasp.webscarab.model.UrlModel;
import org.owasp.webscarab.model.Cookie;
import org.owasp.webscarab.model.NamedValue;
import org.owasp.webscarab.model.HttpUrl;
import org.owasp.webscarab.model.Preferences;
import org.owasp.webscarab.model.FrameworkModel;
import org.owasp.webscarab.model.FilteredUrlModel;
import org.owasp.webscarab.plugin.AbstractPluginModel;
import java.util.List;
import java.util.LinkedList;
import java.util.logging.Logger;

/*
 * SpiderModel.java
 *
 * Created on 04 March 2005, 03:11
 */

package org.owasp.webscarab.plugin.spider;




/**
 *
 * @author  rogan
 */
public class SpiderModel extends AbstractPluginModel {
    
    private FrameworkModel _model;
    private SpiderUrlModel _urlModel;
    
    private List<Link> _linkQueue = new LinkedList<Link>();
    
    private String _allowedDomains = null;
    private String _forbiddenPaths = null;
    private boolean _recursive = false;
    private boolean _cookieSync = false;
    
    private NamedValue[] _extraHeaders = null;
    
    private Logger _logger = Logger.getLogger(getClass().getName());
    
    /** Creates a new instance of SpiderModel */
    public SpiderModel(FrameworkModel model) {
        _model = model;
        _urlModel = new SpiderUrlModel(_model.getUrlModel());
        parseProperties();
    }
    
    public UrlModel getUrlModel() {
        return _urlModel;
    }
    
    public boolean isUnseen(HttpUrl url) {
        return _model.getUrlProperty(url, "METHODS") == null;
    }
    
    public boolean isForbidden(HttpUrl url) {
        if (_forbiddenPaths != null && !_forbiddenPaths.equals("")) {
            try {
                return url.toString().matches(getForbiddenPaths());
            } catch (Exception e) {
            }
        }
        return false;
    }
    
    public void addUnseenLink(HttpUrl url, HttpUrl referer) {
        if (url == null) {
            return;
        }
        if (isUnseen(url)) {
            String first = _model.getUrlProperty(url, "REFERER");
            if (first == null || first.equals("")) {
                _model.setUrlProperty(url, "REFERER", referer.toString());
            }
        }
    }
    
    public void queueLink(Link link) {
        // _logger.info("Queueing " + link);
        try {
            _model.readLock().acquire();
            _linkQueue.add(link);
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
        // _logger.info("Done queuing " + link);
    }
    
    public Link dequeueLink() {
        // _logger.info("Dequeueing a link");
        Link link = null;
        try {
            _model.readLock().acquire();
            if (_linkQueue.size() > 0) 
                link = _linkQueue.remove(0);
            if (_linkQueue.size() == 0) {
                setStatus("Idle");
            } else {
                setStatus(_linkQueue.size() + " links remaining");
            }
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
            // _logger.info("Done dequeuing a link " + link);
        }
        return link;
    }
    
    public void clearLinkQueue() {
        try {
            _model.readLock().acquire();
            _linkQueue.clear();
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
    }
    
    public int getQueuedLinkCount() {
        try {
            _model.readLock().acquire();
            return _linkQueue.size();
        } catch (InterruptedException ie) {
            _logger.warning("Interrupted waiting for the read lock! " + ie.getMessage());
        } finally {
            _model.readLock().release();
        }
        return 0;
    }
    
    public Cookie[] getCookiesForUrl(HttpUrl url) {
        return _model.getCookiesForUrl(url);
    }
    
    public void addCookie(Cookie cookie) {
        _model.addCookie(cookie);
    }
    
    public void parseProperties() {
        String prop = "Spider.domains";
        String value = Preferences.getPreference(prop, ".*localhost.*");