code: def _get_crawled_urls(self, handle, request):
          
          try:
              content = six.text_type(handle.open(request).read(), "utf-8",
                                      errors="replace")
              soup = BeautifulSoup(content, "html.parser")
              tags = soup('a')
              for tag in tqdm(tags):
                  href = tag.get("href")
                  if href is not None:
                      url = urllib.parse.urljoin(self.url, escape(href))
                      if url not in self:
                          self.urls.append(url)
  
          except urllib.request.HTTPError as error:
              if error.code == 404:
                  logger.warning("ERROR: %s -> %s for %s" % (error, error.url, self.url))
              else:
                  logger.warning("ERROR: %s for %s" % (error, self.url))
  
          except urllib.request.URLError as error:
              logger.warning("ERROR: %s for %s" % (error, self.url))
              raise urllib.request.URLError("URL entered is Incorrect")
docs: Main method where the crawler html content is parsed with
          beautiful soup and out of the DOM, we get the urls