code: def _fetch(self):
          
          while True:
              url = self._to_fetch.get()
              try:
                  if url:
                      page = self.get_page(url)
                      if page is None:    # e.g. after an error
                          continue
                      for link, rel in page.links:
                          if link not in self._seen:
                              try:
                                  self._seen.add(link)
                                  if (not self._process_download(link) and
                                      self._should_queue(link, url, rel)):
                                      logger.debug('Queueing %s from %s', link, url)
                                      self._to_fetch.put(link)
                              except MetadataInvalidError:  # e.g. invalid versions
                                  pass
              except Exception as e:  # pragma: no cover
                  self.errors.put(text_type(e))
              finally:
                  # always do this, to avoid hangs :-)
                  self._to_fetch.task_done()
              if not url:
                  #logger.debug('Sentinel seen, quitting.')
                  break
docs: Get a URL to fetch from the work queue, get the HTML page, examine its
          links for download candidates and candidates for further scraping.
  
          This is a handy method to run in a thread.