(self)
Collects links from our root urls, stores them and then calls
        `_browse_from_links` to browse them
Collects links from our root urls, stores them and then calls
        `_browse_from_links` to browse them
Collects links from our root urls stores them and then calls _browse_from_links to browse them
def crawl(self):
        """
        Collects links from our root urls, stores them and then calls
        `_browse_from_links` to browse them
        """
        self._start_time = datetime.datetime.now()

        while True:
            url = random.choice(self._config["root_urls"])
            try:
                body = self._request(url).content
                self._links = self._extract_urls(body, url)
                logging.debug("found {} links".format(len(self._links)))
                self._browse_from_links()

            except requests.exceptions.RequestException:
                logging.warn("Error connecting to root url: {}".format(url))
                
            except MemoryError:
                logging.warn("Error: content at url: {} is exhausting the memory".format(url))

            except LocationParseError:
                logging.warn("Error encountered during parsing of: {}".format(url))

            except self.CrawlerTimedOut:
                logging.info("Timeout has exceeded, exiting")
                return
def crawl ( self ) : self . _start_time = datetime . datetime . now ( ) while True : url = random . choice ( self . _config [ "root_urls" ] ) try : body = self . _request ( url ) . content self . _links = self . _extract_urls ( body , url ) logging . debug ( "found {} links" . format ( len ( self . _links ) ) ) self . _browse_from_links ( ) except requests . exceptions . RequestException : logging . warn ( "Error connecting to root url: {}" . format ( url ) ) except MemoryError : logging . warn ( "Error: content at url: {} is exhausting the memory" . format ( url ) ) except LocationParseError : logging . warn ( "Error encountered during parsing of: {}" . format ( url ) ) except self . CrawlerTimedOut : logging . info ( "Timeout has exceeded, exiting" ) return