code: def allows_recursion (self):
          
          log.debug(LOG_CHECK, "checking recursion of %r ...", self.url)
          if not self.valid:
              log.debug(LOG_CHECK, "... no, invalid.")
              return False
          if not self.can_get_content():
              log.debug(LOG_CHECK, "... no, cannot get content.")
              return False
          if not self.allows_simple_recursion():
              return False
          if self.size > self.aggregate.config["maxfilesizeparse"]:
              log.debug(LOG_CHECK, "... no, maximum parse size.")
              return False
          if not self.is_parseable():
              log.debug(LOG_CHECK, "... no, not parseable.")
              return False
          if not self.content_allows_robots():
              log.debug(LOG_CHECK, "... no, robots.")
              return False
          log.debug(LOG_CHECK, "... yes, recursion.")
          return True
docs: Return True iff we can recurse into the url's content.