(self, crawler=None)
Setup redis connection and idle signal.

        This should be called after the spider has set its crawler object.
Setup redis connection and idle signal.
Setup redis connection and idle signal .
def setup_redis(self, crawler=None):
        """Setup redis connection and idle signal.

        This should be called after the spider has set its crawler object.
        """
        if self.server is not None:
            return

        if crawler is None:
            # We allow optional crawler argument to keep backwards
            # compatibility.
            # XXX: Raise a deprecation warning.
            crawler = getattr(self, 'crawler', None)

        if crawler is None:
            raise ValueError("crawler is required")

        settings = crawler.settings

        if self.redis_key is None:
            self.redis_key = settings.get(
                'REDIS_START_URLS_KEY', defaults.START_URLS_KEY,
            )

        self.redis_key = self.redis_key % {'name': self.name}

        if not self.redis_key.strip():
            raise ValueError("redis_key must not be empty")

        if self.redis_batch_size is None:
            # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
            self.redis_batch_size = settings.getint(
                'REDIS_START_URLS_BATCH_SIZE',
                settings.getint('CONCURRENT_REQUESTS'),
            )

        try:
            self.redis_batch_size = int(self.redis_batch_size)
        except (TypeError, ValueError):
            raise ValueError("redis_batch_size must be an integer")

        if self.redis_encoding is None:
            self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)

        self.logger.info("Reading start URLs from redis key '%(redis_key)s' "
                         "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
                         self.__dict__)

        self.server = connection.from_settings(crawler.settings)
        # The idle signal is called when the spider has no requests left,
        # that's when we will schedule new requests from redis queue
        crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)
def setup_redis ( self , crawler = None ) : if self . server is not None : return if crawler is None : # We allow optional crawler argument to keep backwards # compatibility. # XXX: Raise a deprecation warning. crawler = getattr ( self , 'crawler' , None ) if crawler is None : raise ValueError ( "crawler is required" ) settings = crawler . settings if self . redis_key is None : self . redis_key = settings . get ( 'REDIS_START_URLS_KEY' , defaults . START_URLS_KEY , ) self . redis_key = self . redis_key % { 'name' : self . name } if not self . redis_key . strip ( ) : raise ValueError ( "redis_key must not be empty" ) if self . redis_batch_size is None : # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE). self . redis_batch_size = settings . getint ( 'REDIS_START_URLS_BATCH_SIZE' , settings . getint ( 'CONCURRENT_REQUESTS' ) , ) try : self . redis_batch_size = int ( self . redis_batch_size ) except ( TypeError , ValueError ) : raise ValueError ( "redis_batch_size must be an integer" ) if self . redis_encoding is None : self . redis_encoding = settings . get ( 'REDIS_ENCODING' , defaults . REDIS_ENCODING ) self . logger . info ( "Reading start URLs from redis key '%(redis_key)s' " "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s" , self . __dict__ ) self . server = connection . from_settings ( crawler . settings ) # The idle signal is called when the spider has no requests left, # that's when we will schedule new requests from redis queue crawler . signals . connect ( self . spider_idle , signal = signals . spider_idle )