docs: public void add(String rule, String value) {
          if (rule.equals("sitemap")) {
              if (this.sitemap == null) {
                  this.sitemap = new ArrayList<String>();
              }
              this.sitemap.add(value);
          } else if (rule.equals("crawl-delay")) {
              try {
                  this.crawlDelay = Double.parseDouble(value);
              } catch (NumberFormatException e) {
                  logger.warn("Invalid number format for crawl-delay robots.txt: {}", value);
              }
          } else if (rule.equals("host")) {
              this.preferredHost = value;
          } else if (rule.equals("allow")) {
              this.pathRules.add(new PathRule(HostDirectives.ALLOWED, value));
          } else if (rule.equals("disallow")) {
              this.pathRules.add(new PathRule(HostDirectives.DISALLOWED, value));
          } else {
              logger.error("Invalid key in robots.txt passed to UserAgentRules: {}", rule);
          }
      }