
from __future__ import print_function


def traverse_next(page, next, results):
    """
    Recursive generator to traverse through the next attribute and \
    crawl through the links to be followed
    """
    for link in page.extract_links(next['follow_link']):
        print("Loading page", link.url)
        r = results.copy()
        for attribute in next['scraping'].get('data'):
            if attribute['field'] != "":
                print("\nExtracting", attribute['field'], "attribute", sep=' ')
                r[attribute['field']] = link.extract_content(attribute['selector'], attribute['attr'], attribute['default'])
        if not next['scraping'].get('next'):
            yield r
        else:
            for next2 in next['scraping'].get('next'):
                for result in traverse_next(link, next2, r):
                    yield result


from __future__ import print_function
from colorama import init, Fore, Back

init()


def traverse_next(page, next, results):
    """
    Recursive generator to traverse through the next attribute and \
    crawl through the links to be followed
    """
    for link in page.extract_links(next['follow_link']):
        print(Back.YELLOW + Fore.BLUE + "Loading page ", link.url + Back.RESET + Fore.RESET)
        r = results.copy()
        for attribute in next['scraping'].get('data'):
            if attribute['field'] != "":
                print("\nExtracting", attribute['field'], "attribute", sep=' ')
                r[attribute['field']] = link.extract_content(attribute['selector'], attribute['attr'], attribute['default'])
        if not next['scraping'].get('next'):
            yield r
        else:
            for next2 in next['scraping'].get('next'):
                for result in traverse_next(link, next2, r):
                    yield result

Modify stdout logging in crawler run
Modify stdout logging in crawler run

  
  from __future__ import print_function
+ from colorama import init, Fore, Back
+ 
+ init()
  
  
  def traverse_next(page, next, results):
      """
      Recursive generator to traverse through the next attribute and \
      crawl through the links to be followed
      """
      for link in page.extract_links(next['follow_link']):
-         print("Loading page", link.url)
+         print(Back.YELLOW + Fore.BLUE + "Loading page ", link.url + Back.RESET + Fore.RESET)
          r = results.copy()
          for attribute in next['scraping'].get('data'):
              if attribute['field'] != "":
                  print("\nExtracting", attribute['field'], "attribute", sep=' ')
                  r[attribute['field']] = link.extract_content(attribute['selector'], attribute['attr'], attribute['default'])
          if not next['scraping'].get('next'):
              yield r
          else:
              for next2 in next['scraping'].get('next'):
                  for result in traverse_next(link, next2, r):
                      yield result
  
Modify stdout logging in crawler run
## Code Before:

from __future__ import print_function


def traverse_next(page, next, results):
    """
    Recursive generator to traverse through the next attribute and \
    crawl through the links to be followed
    """
    for link in page.extract_links(next['follow_link']):
        print("Loading page", link.url)
        r = results.copy()
        for attribute in next['scraping'].get('data'):
            if attribute['field'] != "":
                print("\nExtracting", attribute['field'], "attribute", sep=' ')
                r[attribute['field']] = link.extract_content(attribute['selector'], attribute['attr'], attribute['default'])
        if not next['scraping'].get('next'):
            yield r
        else:
            for next2 in next['scraping'].get('next'):
                for result in traverse_next(link, next2, r):
                    yield result

## Instruction:
Modify stdout logging in crawler run
## Code After:

from __future__ import print_function
from colorama import init, Fore, Back

init()


def traverse_next(page, next, results):
    """
    Recursive generator to traverse through the next attribute and \
    crawl through the links to be followed
    """
    for link in page.extract_links(next['follow_link']):
        print(Back.YELLOW + Fore.BLUE + "Loading page ", link.url + Back.RESET + Fore.RESET)
        r = results.copy()
        for attribute in next['scraping'].get('data'):
            if attribute['field'] != "":
                print("\nExtracting", attribute['field'], "attribute", sep=' ')
                r[attribute['field']] = link.extract_content(attribute['selector'], attribute['attr'], attribute['default'])
        if not next['scraping'].get('next'):
            yield r
        else:
            for next2 in next['scraping'].get('next'):
                for result in traverse_next(link, next2, r):
                    yield result
