(self, link, referrer, rel)
return result
Determine whether a link URL from a referring page and with a
        particular "rel" attribute should be queued for scraping.
Determine whether a link URL from a referring page and with a
        particular "rel" attribute should be queued for scraping.
Determine whether a link URL from a referring page and with a particular rel attribute should be queued for scraping .
def _should_queue(self, link, referrer, rel):
        """
        Determine whether a link URL from a referring page and with a
        particular "rel" attribute should be queued for scraping.
        """
        scheme, netloc, path, _, _, _ = urlparse(link)
        if path.endswith(self.source_extensions + self.binary_extensions +
                         self.excluded_extensions):
            result = False
        elif self.skip_externals and not link.startswith(self.base_url):
            result = False
        elif not referrer.startswith(self.base_url):
            result = False
        elif rel not in ('homepage', 'download'):
            result = False
        elif scheme not in ('http', 'https', 'ftp'):
            result = False
        elif self._is_platform_dependent(link):
            result = False
        else:
            host = netloc.split(':', 1)[0]
            if host.lower() == 'localhost':
                result = False
            else:
                result = True
        logger.debug('should_queue: %s (%s) from %s -> %s', link, rel,
                     referrer, result)
        return result
def _should_queue ( self , link , referrer , rel ) : scheme , netloc , path , _ , _ , _ = urlparse ( link ) if path . endswith ( self . source_extensions + self . binary_extensions + self . excluded_extensions ) : result = False elif self . skip_externals and not link . startswith ( self . base_url ) : result = False elif not referrer . startswith ( self . base_url ) : result = False elif rel not in ( 'homepage' , 'download' ) : result = False elif scheme not in ( 'http' , 'https' , 'ftp' ) : result = False elif self . _is_platform_dependent ( link ) : result = False else : host = netloc . split ( ':' , 1 ) [ 0 ] if host . lower ( ) == 'localhost' : result = False else : result = True logger . debug ( 'should_queue: %s (%s) from %s -> %s' , link , rel , referrer , result ) return result