code: def get_sitemap_url(url, allow_subdomains):
          
          if allow_subdomains:
              redirect = UrlExtractor.follow_redirects(
                  "http://" + UrlExtractor.get_allowed_domain(url)
              )
          else:
              redirect = UrlExtractor.follow_redirects(
                  "http://" +
                  UrlExtractor.get_allowed_domain(url, False)
              )
          redirect = UrlExtractor.follow_redirects(url)
  
          # Get robots.txt
          parsed = urlparse(redirect)
          if allow_subdomains:
              url_netloc = parsed.netloc
          else:
              url_netloc = UrlExtractor.get_allowed_domain(
                  parsed.netloc, False)
  
          robots = '{url.scheme}://{url_netloc}/robots.txt'.format(
              url=parsed, url_netloc=url_netloc)
  
          try:
              urllib2.urlopen(robots)
              return robots
          except:
              if allow_subdomains:
                  return UrlExtractor.get_sitemap_url(url, False)
              else:
                  raise Exception('Fatal: no robots.txt found.')
docs: Determines the domain's robot.txt
  
          :param str url: the url to work on
          :param bool allow_subdomains: Determines if the robot.txt may be the
                                        subdomain's
          :return: the robot.txt's address
          :raises Exception: if there's no robot.txt on the site's domain