code: def read (self):
          
          self._reset()
          kwargs = dict(
              headers = {
                  'User-Agent': configuration.UserAgent,
                  'Accept-Encoding': ACCEPT_ENCODING,
              }
          )
          if self.auth:
              kwargs["auth"] = self.auth
          if self.proxies:
              kwargs["proxies"] = self.proxies
          try:
              response = self.session.get(self.url, **kwargs)
              response.raise_for_status()
              content_type = response.headers.get('content-type')
              if content_type and content_type.lower().startswith('text/plain'):
                  self.parse(response.iter_lines())
              else:
                  log.debug(LOG_CHECK, "%r allow all (no text content)", self.url)
                  self.allow_all = True
          except requests.HTTPError as x:
              if x.response.status_code in (401, 403):
                  self.disallow_all = True
                  log.debug(LOG_CHECK, "%r disallow all (code %d)", self.url, x.response.status_code)
              else:
                  self.allow_all = True
                  log.debug(LOG_CHECK, "%r allow all (HTTP error)", self.url)
          except requests.exceptions.Timeout:
              raise
          except requests.exceptions.RequestException:
              # no network or other failure
              self.allow_all = True
              log.debug(LOG_CHECK, "%r allow all (request error)", self.url)
docs: Read the robots.txt URL and feeds it to the parser.