code: public RobotsTxt readRobotsTxt(InputStream inputStream) throws IOException {
      BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
      Group currentGroup = null;
      boolean startGroup = false;
      RobotsTxtImpl robots = new RobotsTxtImpl(matchingStrategy, winningStrategy);
  
      for (Entry entry = readEntry(reader); entry != null; entry = readEntry(reader)) {
        switch (entry.getKey().toUpperCase()) {
          case "USER-AGENT":
            if (!startGroup && currentGroup != null) {
              robots.addGroup(currentGroup);
              currentGroup = null;
            }
  
            if (currentGroup == null) {
              currentGroup = new Group();
            }
  
            currentGroup.addUserAgent(entry.getValue());
            startGroup = true;
            break;
  
          case "DISALLOW":
            if (currentGroup != null) {
              boolean access = entry.getValue().isEmpty();
              currentGroup.addAccess(new Access(currentGroup, entry.getSource(), entry.getValue(), access));
              startGroup = false;
            }
            break;
  
          case "ALLOW":
            if (currentGroup != null) {
              boolean access = !entry.getValue().isEmpty();
              currentGroup.addAccess(new Access(currentGroup, entry.getSource(), entry.getValue(), access));
              startGroup = false;
            }
            break;
  
          case "CRAWL-DELAY":
            if (currentGroup != null) {
              try {
                int crawlDelay = Integer.parseInt(entry.getValue());
                currentGroup.setCrawlDelay(crawlDelay);
                startGroup = false;
              } catch (NumberFormatException ex) {
              }
            } else {
            }
            break;
  
          case "HOST":
            robots.setHost(entry.getValue());
            startGroup = false;
            break;
  
          case "SITEMAP":
            robots.getSitemaps().add(entry.getValue());
            startGroup = false;
            break;
  
          default:
            startGroup = false;
            break;
        }
      }
  
      if (currentGroup != null) {
        robots.addGroup(currentGroup);
      }
  
      return robots;
    }
docs: Reads robots txt.
  
  @param inputStream input stream with robots.txt content.
  @return parsed robots.txt
  @throws IOException if reading stream fails