code: def content_allows_robots (self):
          
          if not self.is_html():
              return True
          # construct parser object
          handler = linkparse.MetaRobotsFinder()
          parser = htmlsax.parser(handler)
          handler.parser = parser
          if self.charset:
              parser.encoding = self.charset
          # parse
          try:
              parser.feed(self.get_content())
              parser.flush()
          except linkparse.StopParse as msg:
              log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
              pass
          # break cyclic dependencies
          handler.parser = None
          parser.handler = None
          return handler.follow
docs: Return False if the content of this URL forbids robots to
          search for recursive links.