code: Stream<Token> tokenize(String input) {
          List<Token> tokens = new ArrayList<Token>();
          int tokenPos = 0;
          while (!input.isEmpty()) {
              boolean matched = false;
              for (Token.Type tokenType : Token.Type.values()) {
                  Matcher matcher = tokenType.pattern.matcher(input);
                  if (matcher.find()) {
                      matched = true;
                      input = matcher.replaceFirst("");
                      if (tokenType != Token.Type.WHITESPACE) {
                          tokens.add(new Token(
                              tokenType,
                              matcher.group(),
                              tokenPos
                          ));
                      }
                      tokenPos += matcher.end();
                      break;
                  }
              }
              if (!matched) {
                  throw new LexerException(input);
              }
          }
          tokens.add(new Token(Token.Type.EOI, null, tokenPos));
          return new Stream<Token>(tokens.toArray(new Token[tokens.size()]));
      }
docs: Tokenizes the specified input string.
  
  @param input the input string to tokenize
  @return a stream of tokens
  @throws LexerException when encounters an illegal character