(self, spider: Spider, reason: str = "cancelled")
return dfd
Close (cancel) spider and clear all its outstanding requests
Close (cancel) spider and clear all its outstanding requests
Close ( cancel ) spider and clear all its outstanding requests
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
        """Close (cancel) spider and clear all its outstanding requests"""
        if self.slot is None:
            raise RuntimeError("Engine slot not assigned")

        if self.slot.closing is not None:
            return self.slot.closing

        logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})

        dfd = self.slot.close()

        def log_failure(msg: str) -> Callable:
            def errback(failure: Failure) -> None:
                logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
            return errback

        dfd.addBoth(lambda _: self.downloader.close())
        dfd.addErrback(log_failure('Downloader close failure'))

        dfd.addBoth(lambda _: self.scraper.close_spider(spider))
        dfd.addErrback(log_failure('Scraper close failure'))

        if hasattr(self.slot.scheduler, "close"):
            dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
            dfd.addErrback(log_failure("Scheduler close failure"))

        dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
            signal=signals.spider_closed, spider=spider, reason=reason,
        ))
        dfd.addErrback(log_failure('Error while sending spider_close signal'))

        dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
        dfd.addErrback(log_failure('Stats close failure'))

        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))

        dfd.addBoth(lambda _: setattr(self, 'slot', None))
        dfd.addErrback(log_failure('Error while unassigning slot'))

        dfd.addBoth(lambda _: setattr(self, 'spider', None))
        dfd.addErrback(log_failure('Error while unassigning spider'))

        dfd.addBoth(lambda _: self._spider_closed_callback(spider))

        return dfd
def close_spider ( self , spider : Spider , reason : str = "cancelled" ) -> Deferred : if self . slot is None : raise RuntimeError ( "Engine slot not assigned" ) if self . slot . closing is not None : return self . slot . closing logger . info ( "Closing spider (%(reason)s)" , { 'reason' : reason } , extra = { 'spider' : spider } ) dfd = self . slot . close ( ) def log_failure ( msg : str ) -> Callable : def errback ( failure : Failure ) -> None : logger . error ( msg , exc_info = failure_to_exc_info ( failure ) , extra = { 'spider' : spider } ) return errback dfd . addBoth ( lambda _ : self . downloader . close ( ) ) dfd . addErrback ( log_failure ( 'Downloader close failure' ) ) dfd . addBoth ( lambda _ : self . scraper . close_spider ( spider ) ) dfd . addErrback ( log_failure ( 'Scraper close failure' ) ) if hasattr ( self . slot . scheduler , "close" ) : dfd . addBoth ( lambda _ : self . slot . scheduler . close ( reason ) ) dfd . addErrback ( log_failure ( "Scheduler close failure" ) ) dfd . addBoth ( lambda _ : self . signals . send_catch_log_deferred ( signal = signals . spider_closed , spider = spider , reason = reason , ) ) dfd . addErrback ( log_failure ( 'Error while sending spider_close signal' ) ) dfd . addBoth ( lambda _ : self . crawler . stats . close_spider ( spider , reason = reason ) ) dfd . addErrback ( log_failure ( 'Stats close failure' ) ) dfd . addBoth ( lambda _ : logger . info ( "Spider closed (%(reason)s)" , { 'reason' : reason } , extra = { 'spider' : spider } ) ) dfd . addBoth ( lambda _ : setattr ( self , 'slot' , None ) ) dfd . addErrback ( log_failure ( 'Error while unassigning slot' ) ) dfd . addBoth ( lambda _ : setattr ( self , 'spider' , None ) ) dfd . addErrback ( log_failure ( 'Error while unassigning spider' ) ) dfd . addBoth ( lambda _ : self . _spider_closed_callback ( spider ) ) return dfd