code: def crawler(site, uri=None):
      
      config = load_site_config(site)
      model = _get_model('crawl', config, uri)
      visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
          site, config
      )
  
      if not visited_set.has(model.hash):
          visited_set.add(model.hash)
          visited_uri_set.add(model.uri)
  
          if (
              model.is_consume_page
              and not consume_set.has(model.hash)
          ):
              consume_set.add(model.hash)
              consume_q.enqueue(
                  consumer,
                  site,
                  model.uri
              )
          else:
              for crawl_uri in model.uris_to_crawl:
                  if (
                      not visited_uri_set.has(crawl_uri)
                      and not crawl_set.has(crawl_uri)
                  ):
                      crawl_set.add(crawl_uri)
                      crawl_q.enqueue(
                          crawler,
                          site,
                          crawl_uri
                      )
docs: Crawl URI using site config.