docs: public boolean shouldVisit(Page referringPage, WebURL url) {
          if (myController.getConfig().isRespectNoFollow()) {
              return !((referringPage != null &&
                      referringPage.getContentType() != null &&
                      referringPage.getContentType().contains("html") &&
                      ((HtmlParseData)referringPage.getParseData())
                          .getMetaTagValue("robots")
                          .contains("nofollow")) ||
                      url.getAttribute("rel").contains("nofollow"));
          }
  
          return true;
      }