import scrapy
from scrapy import log
from scrapy.contrib.spiders import SitemapSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
from rcbi.items import Part

MANUFACTURERS = ["Rctimer", "RCTimer", "BaseCam", "Elgae", "ELGAE", "ArduFlyer", "Boscam", "T-Motor", "HQProp", "Suppo", "Flyduino", "SLS", "Frsky"]
CORRECT = {"Rctimer": "RCTimer", "ELGAE": "Elgae", "Frsky": "FrSky"}
class FlyduinoSpider(SitemapSpider):
    name = "flyduino"
    allowed_domains = ["flyduino.net"]
    sitemap_urls = ["http://flyduino.net/sitemap.xml"]
    
    def parse(self, response):
        item = Part()
        item["site"] = "flyduino"
        item["url"] = response.url
        product_name = response.css("div.hproduct")
        if not product_name:
            return
        item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
        for m in MANUFACTURERS:
          if item["name"].startswith(m):
            if m in CORRECT:
              m = CORRECT[m]
            item["manufacturer"] = m
            item["name"] = item["name"][len(m):].strip()
            break
        return item
import scrapy
from scrapy import log
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from rcbi.items import Part

MANUFACTURERS = ["Rctimer", "RCTimer", "BaseCam", "Elgae", "ELGAE", "ArduFlyer", "Boscam", "T-Motor", "HQProp", "Suppo", "Flyduino", "SLS", "Frsky"]
CORRECT = {"Rctimer": "RCTimer", "ELGAE": "Elgae", "Frsky": "FrSky"}
class FlyduinoSpider(CrawlSpider):
    name = "flyduino"
    allowed_domains = ["flyduino.net"]
    start_urls = ["http://flyduino.net/"]

    rules = (
        # Extract links matching 'category.php' (but not matching 'subsection.php')
        # and follow links from them (since no callback means follow=True by default).
        Rule(LinkExtractor(restrict_css=".categories")),

        # Extract links matching 'item.php' and parse them with the spider's method parse_item
        Rule(LinkExtractor(restrict_css=".article_wrapper h3"), callback='parse_item'),
    )

    def parse_item(self, response):
      item = Part()
      item["site"] = "flyduino"
      item["url"] = response.url
      product_name = response.css("div.hproduct")
      if not product_name:
          return
      item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
      for m in MANUFACTURERS:
        if item["name"].startswith(m):
          if m in CORRECT:
            m = CORRECT[m]
          item["manufacturer"] = m
          item["name"] = item["name"][len(m):].strip()
          break
      return item

Stop using the Flyduino sitemap.
Stop using the Flyduino sitemap.

  import scrapy
  from scrapy import log
- from scrapy.contrib.spiders import SitemapSpider, Rule
+ from scrapy.spiders import CrawlSpider, Rule
- from scrapy.contrib.linkextractors import LinkExtractor
+ from scrapy.linkextractors import LinkExtractor
  from rcbi.items import Part
  
  MANUFACTURERS = ["Rctimer", "RCTimer", "BaseCam", "Elgae", "ELGAE", "ArduFlyer", "Boscam", "T-Motor", "HQProp", "Suppo", "Flyduino", "SLS", "Frsky"]
  CORRECT = {"Rctimer": "RCTimer", "ELGAE": "Elgae", "Frsky": "FrSky"}
- class FlyduinoSpider(SitemapSpider):
+ class FlyduinoSpider(CrawlSpider):
      name = "flyduino"
      allowed_domains = ["flyduino.net"]
-     sitemap_urls = ["http://flyduino.net/sitemap.xml"]
+     start_urls = ["http://flyduino.net/"]
+ 
+     rules = (
+         # Extract links matching 'category.php' (but not matching 'subsection.php')
+         # and follow links from them (since no callback means follow=True by default).
+         Rule(LinkExtractor(restrict_css=".categories")),
+ 
+         # Extract links matching 'item.php' and parse them with the spider's method parse_item
+         Rule(LinkExtractor(restrict_css=".article_wrapper h3"), callback='parse_item'),
-     
+     )
+ 
-     def parse(self, response):
+     def parse_item(self, response):
-         item = Part()
+       item = Part()
-         item["site"] = "flyduino"
+       item["site"] = "flyduino"
-         item["url"] = response.url
+       item["url"] = response.url
-         product_name = response.css("div.hproduct")
+       product_name = response.css("div.hproduct")
-         if not product_name:
+       if not product_name:
-             return
+           return
-         item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
+       item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
-         for m in MANUFACTURERS:
+       for m in MANUFACTURERS:
-           if item["name"].startswith(m):
+         if item["name"].startswith(m):
-             if m in CORRECT:
+           if m in CORRECT:
-               m = CORRECT[m]
+             m = CORRECT[m]
-             item["manufacturer"] = m
+           item["manufacturer"] = m
-             item["name"] = item["name"][len(m):].strip()
+           item["name"] = item["name"][len(m):].strip()
-             break
+           break
-         return item
+       return item
+ 
Stop using the Flyduino sitemap.
## Code Before:
import scrapy
from scrapy import log
from scrapy.contrib.spiders import SitemapSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
from rcbi.items import Part

MANUFACTURERS = ["Rctimer", "RCTimer", "BaseCam", "Elgae", "ELGAE", "ArduFlyer", "Boscam", "T-Motor", "HQProp", "Suppo", "Flyduino", "SLS", "Frsky"]
CORRECT = {"Rctimer": "RCTimer", "ELGAE": "Elgae", "Frsky": "FrSky"}
class FlyduinoSpider(SitemapSpider):
    name = "flyduino"
    allowed_domains = ["flyduino.net"]
    sitemap_urls = ["http://flyduino.net/sitemap.xml"]
    
    def parse(self, response):
        item = Part()
        item["site"] = "flyduino"
        item["url"] = response.url
        product_name = response.css("div.hproduct")
        if not product_name:
            return
        item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
        for m in MANUFACTURERS:
          if item["name"].startswith(m):
            if m in CORRECT:
              m = CORRECT[m]
            item["manufacturer"] = m
            item["name"] = item["name"][len(m):].strip()
            break
        return item
## Instruction:
Stop using the Flyduino sitemap.
## Code After:
import scrapy
from scrapy import log
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from rcbi.items import Part

MANUFACTURERS = ["Rctimer", "RCTimer", "BaseCam", "Elgae", "ELGAE", "ArduFlyer", "Boscam", "T-Motor", "HQProp", "Suppo", "Flyduino", "SLS", "Frsky"]
CORRECT = {"Rctimer": "RCTimer", "ELGAE": "Elgae", "Frsky": "FrSky"}
class FlyduinoSpider(CrawlSpider):
    name = "flyduino"
    allowed_domains = ["flyduino.net"]
    start_urls = ["http://flyduino.net/"]

    rules = (
        # Extract links matching 'category.php' (but not matching 'subsection.php')
        # and follow links from them (since no callback means follow=True by default).
        Rule(LinkExtractor(restrict_css=".categories")),

        # Extract links matching 'item.php' and parse them with the spider's method parse_item
        Rule(LinkExtractor(restrict_css=".article_wrapper h3"), callback='parse_item'),
    )

    def parse_item(self, response):
      item = Part()
      item["site"] = "flyduino"
      item["url"] = response.url
      product_name = response.css("div.hproduct")
      if not product_name:
          return
      item["name"] = product_name[0].xpath("//h1/text()").extract()[0]
      for m in MANUFACTURERS:
        if item["name"].startswith(m):
          if m in CORRECT:
            m = CORRECT[m]
          item["manufacturer"] = m
          item["name"] = item["name"][len(m):].strip()
          break
      return item
