(self, depth=0)
Selects a random link out of the available link list and visits it.
        Blacklists any link that is not responsive or that contains no other links.
        Please note that this function is recursive and will keep calling itself until
        a dead end has reached or when we ran out of links
        :param depth: our current link depth
Selects a random link out of the available link list and visits it.
        Blacklists any link that is not responsive or that contains no other links.
        Please note that this function is recursive and will keep calling itself until
        a dead end has reached or when we ran out of links
        :param depth: our current link depth
Selects a random link out of the available link list and visits it . Blacklists any link that is not responsive or that contains no other links . Please note that this function is recursive and will keep calling itself until a dead end has reached or when we ran out of links : param depth : our current link depth
def _browse_from_links(self, depth=0):
        """
        Selects a random link out of the available link list and visits it.
        Blacklists any link that is not responsive or that contains no other links.
        Please note that this function is recursive and will keep calling itself until
        a dead end has reached or when we ran out of links
        :param depth: our current link depth
        """
        is_depth_reached = depth >= self._config['max_depth']
        if not len(self._links) or is_depth_reached:
            logging.debug("Hit a dead end, moving to the next root URL")
            # escape from the recursion, we don't have links to continue or we have reached the max depth
            return

        if self._is_timeout_reached():
            raise self.CrawlerTimedOut

        random_link = random.choice(self._links)
        try:
            logging.info("Visiting {}".format(random_link))
            sub_page = self._request(random_link).content
            sub_links = self._extract_urls(sub_page, random_link)

            # sleep for a random amount of time
            time.sleep(random.randrange(self._config["min_sleep"], self._config["max_sleep"]))

            # make sure we have more than 1 link to pick from
            if len(sub_links) > 1:
                # extract links from the new page
                self._links = self._extract_urls(sub_page, random_link)
            else:
                # else retry with current link list
                # remove the dead-end link from our list
                self._remove_and_blacklist(random_link)

        except requests.exceptions.RequestException:
            logging.debug("Exception on URL: %s, removing from list and trying again!" % random_link)
            self._remove_and_blacklist(random_link)

        self._browse_from_links(depth + 1)
def _browse_from_links ( self , depth = 0 ) : is_depth_reached = depth >= self . _config [ 'max_depth' ] if not len ( self . _links ) or is_depth_reached : logging . debug ( "Hit a dead end, moving to the next root URL" ) # escape from the recursion, we don't have links to continue or we have reached the max depth return if self . _is_timeout_reached ( ) : raise self . CrawlerTimedOut random_link = random . choice ( self . _links ) try : logging . info ( "Visiting {}" . format ( random_link ) ) sub_page = self . _request ( random_link ) . content sub_links = self . _extract_urls ( sub_page , random_link ) # sleep for a random amount of time time . sleep ( random . randrange ( self . _config [ "min_sleep" ] , self . _config [ "max_sleep" ] ) ) # make sure we have more than 1 link to pick from if len ( sub_links ) > 1 : # extract links from the new page self . _links = self . _extract_urls ( sub_page , random_link ) else : # else retry with current link list # remove the dead-end link from our list self . _remove_and_blacklist ( random_link ) except requests . exceptions . RequestException : logging . debug ( "Exception on URL: %s, removing from list and trying again!" % random_link ) self . _remove_and_blacklist ( random_link ) self . _browse_from_links ( depth + 1 )