code: def _allows_url (self, url_data, roboturl):
          
          with cache_lock:
              if roboturl in self.cache:
                  self.hits += 1
                  rp = self.cache[roboturl]
                  return rp.can_fetch(self.useragent, url_data.url)
              self.misses += 1
          kwargs = dict(auth=url_data.auth, session=url_data.session)
          if hasattr(url_data, "proxy") and hasattr(url_data, "proxy_type"):
              kwargs["proxies"] = {url_data.proxytype: url_data.proxy}
          rp = robotparser2.RobotFileParser(**kwargs)
          rp.set_url(roboturl)
          rp.read()
          with cache_lock:
              self.cache[roboturl] = rp
          self.add_sitemap_urls(rp, url_data, roboturl)
          return rp.can_fetch(self.useragent, url_data.url)
docs: Ask robots.txt allowance. Assumes only single thread per robots.txt
          URL calls this function.