(self)
parse sitemap.xml
        for example: http://www.aouu.com/sitemap.xml
parse sitemap.xml
        for example: http://www.aouu.com/sitemap.xml
parse sitemap . xml for example : http : // www . aouu . com / sitemap . xml
def parseSitemap(self):
        """
        parse sitemap.xml
        for example: http://www.aouu.com/sitemap.xml
        """
        self.task.update_sitemap_parsed('start')
        sitemapUrl = self.domain + "sitemap.xml"
        try:
            response = requests.request(sitemapUrl)
            if not response:
                return
            lines = response.iter_lines() #sitemap.xml may very big
            for line in lines:
                match = SITEMAP_URL.search(line)
                if match:
                    url = match.group('url')
                    if self.isOrigin(url) and url not in self.urlSet:
                        self.urlSet.add(url)
                        self.addRequest(Request.fromUrl(url,sitemapUrl))
        finally:
            self.task.update_sitemap_parsed('finish')
def parseSitemap ( self ) : self . task . update_sitemap_parsed ( 'start' ) sitemapUrl = self . domain + "sitemap.xml" try : response = requests . request ( sitemapUrl ) if not response : return lines = response . iter_lines ( ) #sitemap.xml may very big for line in lines : match = SITEMAP_URL . search ( line ) if match : url = match . group ( 'url' ) if self . isOrigin ( url ) and url not in self . urlSet : self . urlSet . add ( url ) self . addRequest ( Request . fromUrl ( url , sitemapUrl ) ) finally : self . task . update_sitemap_parsed ( 'finish' )