from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess


if __name__ == '__main__':
    settings = get_project_settings()

    settings.set('LOG_FILE', 'all_spiders.log')
    settings.set('LOG_LEVEL', 'ERROR')
    settings.set('TELNETCONSOLE_ENABLED', False)
    settings.set('FEED_URI', 'output.ndgeojson')
    settings.set('FEED_FORMAT', 'ndgeojson')
    settings.get('ITEM_PIPELINES')['locations.pipelines.ApplySpiderNamePipeline'] = 100

    process = CrawlerProcess(settings)
    for spider_name in process.spider_loader.list():
        process.crawl(spider_name)
    process.start()

from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess
from scrapy import signals


if __name__ == '__main__':
    settings = get_project_settings()

    settings.set('LOG_FILE', 'all_spiders.log')
    settings.set('LOG_LEVEL', 'ERROR')
    settings.set('TELNETCONSOLE_ENABLED', False)
    settings.set('FEED_URI', 'output.ndgeojson')
    settings.set('FEED_FORMAT', 'ndgeojson')
    settings.get('ITEM_PIPELINES')['locations.pipelines.ApplySpiderNamePipeline'] = 100

    def spider_opened(spider):
        print("Spider %s opened" % spider.name)

    def spider_closed(spider):
        print("Spider %s closed (%s) after %0.1f sec, %d items" % (
            spider.name,
            spider.crawler.stats.get_value('finish_reason'),
            (spider.crawler.stats.get_value('finish_time') -
                spider.crawler.stats.get_value('start_time')).total_seconds(),
            spider.crawler.stats.get_value('item_scraped_count'),
        ))

    process = CrawlerProcess(settings)
    for spider_name in process.spider_loader.list():
        crawler = process.create_crawler(spider_name)
        crawler.signals.connect(spider_closed, signals.spider_closed)
        crawler.signals.connect(spider_opened, signals.spider_opened)
        process.crawl(crawler)
    process.start()

Print some stats in the crawler
Print some stats in the crawler

  from scrapy.utils.project import get_project_settings
  from scrapy.crawler import CrawlerProcess
+ from scrapy import signals
  
  
  if __name__ == '__main__':
      settings = get_project_settings()
  
      settings.set('LOG_FILE', 'all_spiders.log')
      settings.set('LOG_LEVEL', 'ERROR')
      settings.set('TELNETCONSOLE_ENABLED', False)
      settings.set('FEED_URI', 'output.ndgeojson')
      settings.set('FEED_FORMAT', 'ndgeojson')
      settings.get('ITEM_PIPELINES')['locations.pipelines.ApplySpiderNamePipeline'] = 100
  
+     def spider_opened(spider):
+         print("Spider %s opened" % spider.name)
+ 
+     def spider_closed(spider):
+         print("Spider %s closed (%s) after %0.1f sec, %d items" % (
+             spider.name,
+             spider.crawler.stats.get_value('finish_reason'),
+             (spider.crawler.stats.get_value('finish_time') -
+                 spider.crawler.stats.get_value('start_time')).total_seconds(),
+             spider.crawler.stats.get_value('item_scraped_count'),
+         ))
+ 
      process = CrawlerProcess(settings)
      for spider_name in process.spider_loader.list():
-         process.crawl(spider_name)
+         crawler = process.create_crawler(spider_name)
+         crawler.signals.connect(spider_closed, signals.spider_closed)
+         crawler.signals.connect(spider_opened, signals.spider_opened)
+         process.crawl(crawler)
      process.start()
  
Print some stats in the crawler
## Code Before:
from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess


if __name__ == '__main__':
    settings = get_project_settings()

    settings.set('LOG_FILE', 'all_spiders.log')
    settings.set('LOG_LEVEL', 'ERROR')
    settings.set('TELNETCONSOLE_ENABLED', False)
    settings.set('FEED_URI', 'output.ndgeojson')
    settings.set('FEED_FORMAT', 'ndgeojson')
    settings.get('ITEM_PIPELINES')['locations.pipelines.ApplySpiderNamePipeline'] = 100

    process = CrawlerProcess(settings)
    for spider_name in process.spider_loader.list():
        process.crawl(spider_name)
    process.start()

## Instruction:
Print some stats in the crawler
## Code After:
from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess
from scrapy import signals


if __name__ == '__main__':
    settings = get_project_settings()

    settings.set('LOG_FILE', 'all_spiders.log')
    settings.set('LOG_LEVEL', 'ERROR')
    settings.set('TELNETCONSOLE_ENABLED', False)
    settings.set('FEED_URI', 'output.ndgeojson')
    settings.set('FEED_FORMAT', 'ndgeojson')
    settings.get('ITEM_PIPELINES')['locations.pipelines.ApplySpiderNamePipeline'] = 100

    def spider_opened(spider):
        print("Spider %s opened" % spider.name)

    def spider_closed(spider):
        print("Spider %s closed (%s) after %0.1f sec, %d items" % (
            spider.name,
            spider.crawler.stats.get_value('finish_reason'),
            (spider.crawler.stats.get_value('finish_time') -
                spider.crawler.stats.get_value('start_time')).total_seconds(),
            spider.crawler.stats.get_value('item_scraped_count'),
        ))

    process = CrawlerProcess(settings)
    for spider_name in process.spider_loader.list():
        crawler = process.create_crawler(spider_name)
        crawler.signals.connect(spider_closed, signals.spider_closed)
        crawler.signals.connect(spider_opened, signals.spider_opened)
        process.crawl(crawler)
    process.start()
