from scrapy.exceptions import DropItem


class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        ref = item['ref']
        if ref in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(ref)
            return item

from scrapy.exceptions import DropItem


class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        ref = item['ref']
        if ref in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(ref)
            return item


class ApplySpiderNamePipeline(object):

    def process_item(self, item, spider):
        existing_extras = item.get('extras', {})
        existing_extras['@spider'] = spider.name
        item['extras'] = existing_extras

        return item

Add pipeline step that adds spider name to properties
Add pipeline step that adds spider name to properties

  from scrapy.exceptions import DropItem
  
  
  class DuplicatesPipeline(object):
  
      def __init__(self):
          self.ids_seen = set()
  
      def process_item(self, item, spider):
          ref = item['ref']
          if ref in self.ids_seen:
              raise DropItem("Duplicate item found: %s" % item)
          else:
              self.ids_seen.add(ref)
              return item
  
+ 
+ class ApplySpiderNamePipeline(object):
+ 
+     def process_item(self, item, spider):
+         existing_extras = item.get('extras', {})
+         existing_extras['@spider'] = spider.name
+         item['extras'] = existing_extras
+ 
+         return item
+ 
Add pipeline step that adds spider name to properties
## Code Before:
from scrapy.exceptions import DropItem


class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        ref = item['ref']
        if ref in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(ref)
            return item

## Instruction:
Add pipeline step that adds spider name to properties
## Code After:
from scrapy.exceptions import DropItem


class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        ref = item['ref']
        if ref in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(ref)
            return item


class ApplySpiderNamePipeline(object):

    def process_item(self, item, spider):
        existing_extras = item.get('extras', {})
        existing_extras['@spider'] = spider.name
        item['extras'] = existing_extras

        return item
