code: public void parse(InputStream is) throws IOException {
  		
  		BufferedReader br = new BufferedReader(new InputStreamReader(
  				(InputStream) is));
          String read;
          ArrayList<String> current = null;
          while (br != null) {
              do {
                  read = br.readLine();
                  // Skip comments & blanks
              } while ((read != null) && ((read = read.trim()).startsWith("#") ||
                  read.length() == 0));
              if (read == null) {
              	br.close();
              	br = null;
              } else {
                  int commentIndex = read.indexOf("#");
                  if (commentIndex > -1) {
                      // Strip trailing comment
                      read = read.substring(0, commentIndex);
                  }
                  read = read.trim();
                  if (read.matches("(?i)^User-agent:.*")) {
                      String ua = read.substring(11).trim().toLowerCase();
                      if (current == null || current.size() != 0) {
                          // only create new rules-list if necessary
                          // otherwise share with previous user-agent
                          current = new ArrayList<String>();
                      }
                      rules.put(ua, current);
                      LOGGER.fine("Found User-agent(" + ua + ") rules...");
                      continue;
                  }
                  if (read.matches("(?i)Disallow:.*")) {
                      if (current == null) {
                          // buggy robots.txt
                      	bSyntaxErrors = true;
                          continue;
                      }
                      String path = read.substring(9).trim();
                      current.add(path);
                      continue;
                  }
                  // unknown line; do nothing for now
                  
                  // TODO: check for "Allow" lines, and flag a syntax error if 
                  //       we encounter any unknown lines?
              }
          }
      }
docs: Read rules from InputStream argument into this RobotRules, as a
  side-effect, sets the bSyntaxErrors property.
  
  @param is
  @throws IOException