from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "Penny Arcade"
    language = "en"
    url = "http://penny-arcade.com/"
    start_date = "1998-11-18"
    rights = "Mike Krahulik & Jerry Holkins"


class Crawler(CrawlerBase):
    history_capable_date = "1998-11-18"
    schedule = "Mo,We,Fr"
    time_zone = "US/Pacific"

    # Without User-Agent set, the server returns 403 Forbidden
    headers = {"User-Agent": "Mozilla/4.0"}

    def crawl(self, pub_date):
        page_url = "http://penny-arcade.com/comic/%s" % (
            pub_date.strftime("%Y/%m/%d"),
        )
        page = self.parse_page(page_url)
        title = page.alt("#comicFrame img")
        url = page.src("#comicFrame img")
        return CrawlerImage(url, title)

from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "Penny Arcade"
    language = "en"
    url = "http://penny-arcade.com/"
    start_date = "1998-11-18"
    rights = "Mike Krahulik & Jerry Holkins"


class Crawler(CrawlerBase):
    history_capable_date = "1998-11-18"
    schedule = "Mo,We,Fr"
    time_zone = "US/Pacific"

    # Without User-Agent set, the server returns 403 Forbidden
    headers = {"User-Agent": "Mozilla/4.0"}

    def crawl(self, pub_date):
        page_url = "http://penny-arcade.com/comic/%s" % (
            pub_date.strftime("%Y/%m/%d"),
        )
        page = self.parse_page(page_url)
        # The site gives a 404 page without a real 404 code
        page_title = page.text("title")
        if page_title == "Penny Arcade - 404":
            return

        title = page.alt("#comicFrame img")
        url = page.src("#comicFrame img")
        return CrawlerImage(url, title)

Check "Penny Arcade" for 404 page without 404 header
Check "Penny Arcade" for 404 page without 404 header

  from comics.aggregator.crawler import CrawlerBase, CrawlerImage
  from comics.core.comic_data import ComicDataBase
  
  
  class ComicData(ComicDataBase):
      name = "Penny Arcade"
      language = "en"
      url = "http://penny-arcade.com/"
      start_date = "1998-11-18"
      rights = "Mike Krahulik & Jerry Holkins"
  
  
  class Crawler(CrawlerBase):
      history_capable_date = "1998-11-18"
      schedule = "Mo,We,Fr"
      time_zone = "US/Pacific"
  
      # Without User-Agent set, the server returns 403 Forbidden
      headers = {"User-Agent": "Mozilla/4.0"}
  
      def crawl(self, pub_date):
          page_url = "http://penny-arcade.com/comic/%s" % (
              pub_date.strftime("%Y/%m/%d"),
          )
          page = self.parse_page(page_url)
+         # The site gives a 404 page without a real 404 code
+         page_title = page.text("title")
+         if page_title == "Penny Arcade - 404":
+             return
+ 
          title = page.alt("#comicFrame img")
          url = page.src("#comicFrame img")
          return CrawlerImage(url, title)
  
Check "Penny Arcade" for 404 page without 404 header
## Code Before:
from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "Penny Arcade"
    language = "en"
    url = "http://penny-arcade.com/"
    start_date = "1998-11-18"
    rights = "Mike Krahulik & Jerry Holkins"


class Crawler(CrawlerBase):
    history_capable_date = "1998-11-18"
    schedule = "Mo,We,Fr"
    time_zone = "US/Pacific"

    # Without User-Agent set, the server returns 403 Forbidden
    headers = {"User-Agent": "Mozilla/4.0"}

    def crawl(self, pub_date):
        page_url = "http://penny-arcade.com/comic/%s" % (
            pub_date.strftime("%Y/%m/%d"),
        )
        page = self.parse_page(page_url)
        title = page.alt("#comicFrame img")
        url = page.src("#comicFrame img")
        return CrawlerImage(url, title)

## Instruction:
Check "Penny Arcade" for 404 page without 404 header
## Code After:
from comics.aggregator.crawler import CrawlerBase, CrawlerImage
from comics.core.comic_data import ComicDataBase


class ComicData(ComicDataBase):
    name = "Penny Arcade"
    language = "en"
    url = "http://penny-arcade.com/"
    start_date = "1998-11-18"
    rights = "Mike Krahulik & Jerry Holkins"


class Crawler(CrawlerBase):
    history_capable_date = "1998-11-18"
    schedule = "Mo,We,Fr"
    time_zone = "US/Pacific"

    # Without User-Agent set, the server returns 403 Forbidden
    headers = {"User-Agent": "Mozilla/4.0"}

    def crawl(self, pub_date):
        page_url = "http://penny-arcade.com/comic/%s" % (
            pub_date.strftime("%Y/%m/%d"),
        )
        page = self.parse_page(page_url)
        # The site gives a 404 page without a real 404 code
        page_title = page.text("title")
        if page_title == "Penny Arcade - 404":
            return

        title = page.alt("#comicFrame img")
        url = page.src("#comicFrame img")
        return CrawlerImage(url, title)
