code: def can_fetch (self, useragent, url):
          
          log.debug(LOG_CHECK, "%r check allowance for:\n  user agent: %r\n  url: %r ...", self.url, useragent, url)
          if not isinstance(useragent, str):
              useragent = useragent.encode("ascii", "ignore")
          if not isinstance(url, str):
              url = url.encode("ascii", "ignore")
          if self.disallow_all:
              log.debug(LOG_CHECK, " ... disallow all.")
              return False
          if self.allow_all:
              log.debug(LOG_CHECK, " ... allow all.")
              return True
          # search for given user agent matches
          # the first match counts
          url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
          for entry in self.entries:
              if entry.applies_to(useragent):
                  return entry.allowance(url)
          # try the default entry last
          if self.default_entry is not None:
              return self.default_entry.allowance(url)
          # agent not found ==> access granted
          log.debug(LOG_CHECK, " ... agent not found, allow.")
          return True
docs: Using the parsed robots.txt decide if useragent can fetch url.
  
          @return: True if agent can fetch url, else False
          @rtype: bool