# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


def serialize_rank_popularity(value):
    return int(str(value).strip()[1: ])


class MyanimelistScraperItem(scrapy.Item):
    # define the fields for your item here like:
    
    # anime
    en_name = scrapy.Field()
    jp_name = scrapy.Field()
    rank = scrapy.Field(serializer=serialize_rank_popularity)
    popularity = scrapy.Field(serializer=serialize_rank_popularity)
    image = scrapy.Field()
    description = scrapy.Field()
    genre = scrapy.Field()
    score = scrapy.Field()
    url = scrapy.Field()
    members = scrapy.Field()
    favorites = scrapy.Field()
    rating = scrapy.Field()
    duration = scrapy.Field()
    aired = scrapy.Field()
    episodes = scrapy.Field()
    scurce = scrapy.Field()
    scored_by = scrapy.Field()
    studios = scrapy.Field()

    # reviews
    reviews = scrapy.Field()
    
    # characters
    characters_information = scrapy.Field()
myanimelist_scraper/items.py
v3xlrm1nOwo1/myanimelist-scraper
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter


class Myanimelist_scraperSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class MyanimelistScraperDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


myanimelist_scraper/middlewares.py
v3xlrm1nOwo1/myanimelist-scraper
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import re

class MyanimelistScraperPipeline:
    def process_item(self, item, spider):
        
        adapter = ItemAdapter(item=item)
        field_names = adapter.field_names()
        
        for field_name in field_names:
            
            if field_name == 'description':
                value = adapter.get(field_name)
                description = ''
                for row in value:
                    row = row.strip()
                    if row.find('[Written by') == -1:
                        description += row + ' '
                    else:
                        description = description
                adapter[field_name] = description.strip()
                

            elif field_name == 'score':
                value = adapter.get(field_name)
                
                adapter[field_name] = float(value)

            
            elif field_name == 'members' or field_name == 'favorites':
                value = adapter.get(field_name)
                value = value[0].strip().split(',')
                int_value = ''
                for val in value:
                    int_value += val
                adapter[field_name] = int(int_value)


            elif field_name == 'rating' or field_name == 'duration' or field_name == 'aired' or field_name == 'scurce':
                value = adapter.get(field_name)
                adapter[field_name] = value[0].strip()

            
            elif field_name == 'scored_by' or field_name == 'episodes':
                value = adapter.get(field_name)
                adapter[field_name] = int(str(value[0]).strip())


            elif field_name == 'studios':
                value = adapter.get(field_name)
                adapter[field_name] = value

            
            elif field_name == 'reviews':
                review_list = adapter.get(field_name)

                new_review_list = list()

                for review_dict in review_list:
                    new_review_dict = dict()

                    for key, value in review_dict.items():

                        if key == 'username' or key == 'user_img' or key == 'user_link' or key == 'tag':
                            new_review_dict[key] = value.strip()

                        elif key == 'reviewers_rating':
                            new_review_dict[key] = int(value.strip())

                        elif key == 'review_text':
                            new_review_dict[key] = self._clean_text(value)

                        else:
                            new_review_dict[key] = value
                        
                    new_review_list.append(new_review_dict)
                adapter[field_name] = new_review_list


            elif field_name == 'characters_information':
                characters_info_list = adapter.get(field_name)

                new_characters_info_list = list()

                for character_info_dict in characters_info_list:

                    new_character_info_dict = dict()

                    for key, value in character_info_dict.items():
                        
                        if key == 'image' or key == 'en_name' or key == 'jp_name':
                            new_character_info_dict[key] = value.strip()

                        elif key == 'char_description':
                            new_character_info_dict[key] = self._clean_text(value)

                        else:
                            new_character_info_dict[key] = value

                    new_characters_info_list.append(new_character_info_dict)
                adapter[field_name] = new_characters_info_list

        return item
    
    
    def _clean_text(self, text):
        clean_text = re.compile(r'<.*?>')
        return clean_text.sub('', text).strip()
    

myanimelist_scraper/pipelines.py
v3xlrm1nOwo1/myanimelist-scraper
# Scrapy settings for myanimelist_scraper project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from . import config
BOT_NAME = "myanimelist_scraper"

SPIDER_MODULES = ["myanimelist_scraper.spiders"]
NEWSPIDER_MODULE = "myanimelist_scraper.spiders"

# settings.py 

FEEDS = {
    'anime_data.json': {'format': 'json', 'overwrite': True}
}


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = "myanimelist_scraper (+http://www.yourdomain.com)"

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
#    "Accept-Language": "en",
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    "myanimelist_scraper.middlewares.MyanimelistScraperSpiderMiddleware": 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    "myanimelist_scraper.middlewares.MyanimelistScraperDownloaderMiddleware": 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    "scrapy.extensions.telnet.TelnetConsole": None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   "myanimelist_scraper.pipelines.MyanimelistScraperPipeline": 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = "httpcache"
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"

# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"


SCRAPEOPS_API_KEY = config.API_KEY
SCRAPEOPS_PROXY_ENABLED = True
SCRAPEOPS_PROXY_SETTINGS = {'country': 'us'}
DOWNLOADER_MIDDLEWARES = {
    'scrapeops_scrapy_proxy_sdk.scrapeops_scrapy_proxy_sdk.ScrapeOpsScrapyProxySdk': 725,
}


myanimelist_scraper/settings.py
v3xlrm1nOwo1/myanimelist-scraper
import scrapy
from myanimelist_scraper.items import MyanimelistScraperItem
from scrapy.utils.defer import maybe_deferred_to_future



class MyanimelistSpiderSpider(scrapy.Spider):
    name = 'myanimelist_spider'
    allowed_domains = ['myanimelist.net', 'proxy.scrapeops.io']
    start_urls = ['https://myanimelist.net/topanime.php']

    
    custom_settings = {
        'FEEDS': {
            'anime_data.json': { 'format': 'json', 'overwrite': True},
            }
        }


    def parse(self, response):
        table = response.css('table.top-ranking-table')[0]
        all_anime = table.css('tr.ranking-list')
        
        for anime in all_anime:
            anime_url = str(anime.css('td div.detail div h3 a  ::attr(href)').get()).strip()
            if anime_url is not None:
                yield response.follow(url=anime_url, callback=self.parse_anime_page)
        
        
        next_page = response.css('div.pagination a.next ::attr(href)').get()
        
        if next_page is not None:
            next_page_url = f'https://myanimelist.net/topanime.php{next_page}'
            yield response.follow(url=next_page_url, callback=self.parse)

    # get all anime information
    def parse_anime_page(self, response):
        row_table = response.css('div#contentWrapper div#content table tr')

        url = response.url

        resp_meta = {
            'rank': str(row_table.css('td div.pb16 div.di-t div div div.di-ib span.ranked strong ::text').get()).strip(),
            'popularity': str(row_table.css('td div.pb16 div.di-t div div div.di-ib span.popularity strong ::text').get()).strip(),
            'en_name': str(response.css('div div div.h1 div.h1-title div h1.title-name strong ::text').get()).strip(),
            'jp_name': [key.get() for key in [key.css(' ::text') for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Japanese:'][0] if str(key.get()).strip() != '' and str(key.get()).strip() != ',' and str(key.get()).strip() != 'Japanese:'][0].strip(),
            'image': row_table.css('td.borderClass div.leftside div a img[itemprop="image"]::attr(data-src)').get().strip(),
            'description': [des.get() for des in  row_table.css('td div table tr td p[itemprop="description"]::text')],
            'genre': [genre.css('::text').get() for genre in row_table.css('td.borderClass div.leftside div span[itemprop="genre"]')],
            'score': row_table.css('td div.pb16 div.di-t div div div div.score-label::text').get(),
            'members': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Members:'],
            'favorites': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Favorites:'],
            'rating': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Rating:'],
            'duration': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Duration:'],
            'aired': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Aired:'],
            'episodes': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Episodes:'],
            'scurce': [key.css(' ::text')[-1].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Source:'],
            'scored_by': [key.css(' ::text')[6].get() for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Score:'],
            'studios': [key.get() for key in [key.css(' ::text') for key in row_table.css('td.borderClass div.leftside div.spaceit_pad') if key.css('span.dark_text ::text').get() == 'Studios:'][0] if str(key.get()).strip() != '' and str(key.get()).strip() != ',' and str(key.get()).strip() != 'Studios:'],
            'url': str(url).strip()
            }

        # go to reviews
        reviews_page_url = f'{url}/reviews&spoiler=on'
        yield response.follow(url=reviews_page_url, callback=self.parse_get_reviews, meta=resp_meta)
    

    # get all anime reviews
    def parse_get_reviews(self, response):

        reviews_data = response.css('div[id="contentWrapper"] div[id="content"] table tr td[valign="top"] div[class="rightside js-scrollfix-bottom-rel"] div[class="review-element js-review-element"] div[class="thumbbody mt8"]')

        reviews = []
        for review_data in reviews_data:

            resp_review_data = {
                'username': str(review_data.css('div.body div.username a::text').get()).strip(),
                'user_img': str(review_data.css('div.thumb a img::attr(src)').get()).strip(),
                'user_link': str(review_data.css('div.body div.username a::attr(href)').get()).strip(),
                'review_text': ''.join(review_data.css('div.body div.text').getall()).strip(),
                'tag': str(review_data.css('div.body div.tags div::text').get()).strip(),
                'reviewers_rating': str(review_data.css('div.body div[class="rating mt20 mb20 js-hidden"] span.num ::text').get()).strip(),
                'reactions': [(reaction.css('::text').get(), reaction.css('span.num ::text').get(), reaction.css('img::attr(src)').get()) for reaction in review_data.css('div.body div.reaction-box div')],
            }
            
            reviews.append(resp_review_data)
            
        resp_meta = {
            **response.request.meta,
            'reviews': reviews
        }

        # next reviews page
        next_reviews_page = response.css('div[id="contentWrapper"] div[id="content"] table tr td[valign="top"] div.mt4 div[class="ml4 mb8"] a ::attr(href)').get()
        if next_reviews_page is not None:
            yield response.follow(url=next_reviews_page, callback=self.parse_get_reviews)
        
        else:
            # go to characters and voice actors
            try:
                characters_and_voice_actors_page_url = f'{response.request.meta["url"]}/characters'
                yield response.follow(url=characters_and_voice_actors_page_url, callback=self.parse_get_characters_and_voice_actors, meta=resp_meta)
            except:
                print(f'*************!!!!None Data**************\n\n{characters_and_voice_actors_page_url}\n\n{response.request.meta}\n\n')


    # get all anime characters and voice actors
    async def parse_get_characters_and_voice_actors(self, response):
        resp_meta = {
            **response.request.meta,
            'characters_information': [],
            }
        
        characters_url = response.css('table[class="js-anime-character-table"] tr td[valign="top"] div.spaceit_pad a::attr(href)').getall()
        characters_url = [char_url for char_url in characters_url if char_url.find('character') != -1]

        for idx, char_url in enumerate(characters_url):
            
            if char_url is not None:
                request = scrapy.Request(char_url.strip())
                response = await maybe_deferred_to_future(self.crawler.engine.download(request))
                char_information = response.css('table tr td[valign="top"]::text').getall()
                char_information = [char_info.strip() for char_info in char_information if char_info.strip() != '']

                all_data = response.css('table tr td[valign="top"]')
                all_data = all_data.css('::text').getall()
                all_data = [data.strip() for data in all_data if data.strip() != '']

                all_char_information = all_data[all_data.index(char_information[1]): all_data.index(char_information[-1]) + 1]
                all_char_information.insert(0, char_information[0])

                char_information_dict = {'char_description': ''}

                for idx, char_info in enumerate(all_char_information):
                    char_info = char_info.strip()
                    if char_info != '':
                        char_info = char_info.split(': ')
                        if len(char_info) == 2:
                            char_information_dict['_'.join(char_info[0].strip().lower().split()).strip()] = char_info[1].strip()
                        else:
                            char_information_dict['char_description'] += ' ' + char_info[0].strip()

                # voice actors data
                voice_actors_data = response.css('table table')[2: ]
                voice_actors_data_list = []
                for idx, voice_actor in enumerate(voice_actors_data):
                    voice_actor_data = {
                        'name': str(voice_actor.css('tr td.borderClass a::text').get()).strip(),
                        'image': str(voice_actor.css('tr td div.picSurround a img::attr(data-src)').get()).strip(),
                        'country': str(voice_actor.css('tr td.borderClass div small::text').get()).strip(),
                        }
                    voice_actors_data_list.append(voice_actor_data)

                # all character information
                char = {
                    'image': str(response.css('table tr td.borderClass div a img::attr(data-src)').get()).strip(),
                    'en_name': str(response.css('table tr td[valign="top"] h2.normal_header::text').get()).strip(),
                    'jp_name': str(response.css('table tr td[valign="top"] h2.normal_header span small::text').get()).strip(),
                    **char_information_dict,
                    'voice_actors': voice_actors_data_list,
                    }
                
                resp_meta['characters_information'].append({str(response.css('table tr td[valign="top"] h2.normal_header::text').get()).strip(): char})

        yield response.follow(url='None', callback=self.parse_combine_data, meta=resp_meta)


    # combine all data
    def parse_combine_data(self, response):

        # anime data
        anime_item = MyanimelistScraperItem()
        anime_item['rank'] = response.request.meta['rank']
        anime_item['popularity'] = response.request.meta['popularity']
        anime_item['en_name'] = response.request.meta['en_name']
        anime_item['jp_name'] = response.request.meta['jp_name']
        anime_item['image'] = response.request.meta['image']
        anime_item['description'] = response.request.meta['description']
        anime_item['genre'] = response.request.meta['genre']
        anime_item['score'] = response.request.meta['score']
        anime_item['members'] = response.request.meta['members']
        anime_item['favorites'] = response.request.meta['favorites']
        anime_item['rating'] = response.request.meta['rating']
        anime_item['duration'] = response.request.meta['duration']
        anime_item['aired'] = response.request.meta['aired']
        anime_item['episodes'] = response.request.meta['episodes']
        anime_item['scurce'] = response.request.meta['scurce']
        anime_item['scored_by'] = response.request.meta['scored_by']
        anime_item['studios'] = response.request.meta['studios']
        anime_item['url'] = response.request.meta['url']

        # review data
        anime_item['reviews'] = response.request.meta['reviews']

        # character data
        anime_item['characters_information'] = response.request.meta['characters_information']

        yield anime_item

 
myanimelist_scraper/spiders/myanimelist_spider.py
v3xlrm1nOwo1/myanimelist-scraper