(self)
Get a URL to fetch from the work queue, get the HTML page, examine its
        links for download candidates and candidates for further scraping.

        This is a handy method to run in a thread.
Get a URL to fetch from the work queue, get the HTML page, examine its
        links for download candidates and candidates for further scraping.
Get a URL to fetch from the work queue get the HTML page examine its links for download candidates and candidates for further scraping .
def _fetch(self):
        """
        Get a URL to fetch from the work queue, get the HTML page, examine its
        links for download candidates and candidates for further scraping.

        This is a handy method to run in a thread.
        """
        while True:
            url = self._to_fetch.get()
            try:
                if url:
                    page = self.get_page(url)
                    if page is None:    # e.g. after an error
                        continue
                    for link, rel in page.links:
                        if link not in self._seen:
                            try:
                                self._seen.add(link)
                                if (not self._process_download(link) and
                                    self._should_queue(link, url, rel)):
                                    logger.debug('Queueing %s from %s', link, url)
                                    self._to_fetch.put(link)
                            except MetadataInvalidError:  # e.g. invalid versions
                                pass
            except Exception as e:  # pragma: no cover
                self.errors.put(text_type(e))
            finally:
                # always do this, to avoid hangs :-)
                self._to_fetch.task_done()
            if not url:
                #logger.debug('Sentinel seen, quitting.')
                break
def _fetch ( self ) : while True : url = self . _to_fetch . get ( ) try : if url : page = self . get_page ( url ) if page is None : # e.g. after an error continue for link , rel in page . links : if link not in self . _seen : try : self . _seen . add ( link ) if ( not self . _process_download ( link ) and self . _should_queue ( link , url , rel ) ) : logger . debug ( 'Queueing %s from %s' , link , url ) self . _to_fetch . put ( link ) except MetadataInvalidError : # e.g. invalid versions pass except Exception as e : # pragma: no cover self . errors . put ( text_type ( e ) ) finally : # always do this, to avoid hangs :-) self . _to_fetch . task_done ( ) if not url : #logger.debug('Sentinel seen, quitting.') break