docs: private HtmlToken parseToken() {
      int start = offset;
      int limit = input.length();
      if (start == limit) { return null; }
  
      int end = start + 1;
      HtmlTokenType type;
  
      char ch = input.charAt(start);
      if (inTag) {
        if ('>' == ch) {
          type = HtmlTokenType.TAGEND;
          inTag = false;
        } else if ('/' == ch) {
          if (end != limit && '>' == input.charAt(end)) {
            type = HtmlTokenType.TAGEND;
            inTag = false;
            ++end;
          } else {
            type = HtmlTokenType.TEXT;
          }
        } else if ('=' == ch) {
          type = HtmlTokenType.TEXT;
        } else if ('"' == ch || '\'' == ch) {
          type = HtmlTokenType.QSTRING;
          int delim = ch;
          for (; end < limit; ++end) {
            if (input.charAt(end) == delim) {
              ++end;
              break;
            }
          }
        } else if (!Character.isWhitespace(ch)) {
          type = HtmlTokenType.TEXT;
          for (; end < limit; ++end) {
            ch = input.charAt(end);
            // End a text chunk before />
            if ((lastNonIgnorable == null
                 || !lastNonIgnorable.tokenInContextMatches(input, "="))
                && '/' == ch && end + 1 < limit
                && '>' == input.charAt(end + 1)) {
              break;
            } else if ('>' == ch || '=' == ch
                       || Character.isWhitespace(ch)) {
              break;
            } else if ('"' == ch || '\'' == ch) {
              if (end + 1 < limit) {
                char ch2 = input.charAt(end + 1);
                if (Character.isWhitespace(ch2)
                    || ch2 == '>' || ch2 == '/') {
                  ++end;
                  break;
                }
              }
            }
          }
        } else {
          // We skip whitespace tokens inside tag bodies.
          type = HtmlTokenType.IGNORABLE;
          while (end < limit && Character.isWhitespace(input.charAt(end))) {
            ++end;
          }
        }
      } else {
        if (ch == '<') {
          if (end == limit) {
            type = HtmlTokenType.TEXT;
          } else {
            ch = input.charAt(end);
            type = null;
            State state = null;
            switch (ch) {
              case '/':  // close tag?
                state = State.SLASH;
                ++end;
                break;
              case '!':  // Comment or declaration
                if (!this.inEscapeExemptBlock) {
                  state = State.BANG;
                }
                ++end;
                break;
              case '?':
                if (!this.inEscapeExemptBlock) {
                  state = State.BOGUS_COMMENT;
                }
                ++end;
                break;
              case '%':
                state = State.SERVER_CODE;
                ++end;
                break;
              default:
                if (isIdentStart(ch) && !this.inEscapeExemptBlock) {
                  state = State.TAGNAME;
                  ++end;
                } else if ('<' == ch) {
                  type = HtmlTokenType.TEXT;
                } else {
                  ++end;
                }
                break;
            }
            if (null != state) {
              charloop:
              while (end < limit) {
                ch = input.charAt(end);
                switch (state) {
                  case TAGNAME:
                    if (Character.isWhitespace(ch)
                        || '>' == ch || '/' == ch || '<' == ch) {
                      // End processing of an escape exempt block when we see
                      // a corresponding end tag.
                      if (this.inEscapeExemptBlock
                          && '/' == input.charAt(start + 1)
                          && textEscapingMode != HtmlTextEscapingMode.PLAIN_TEXT
                          && canonicalName(start + 2, end)
                              .equals(escapeExemptTagName)) {
                        this.inEscapeExemptBlock = false;
                        this.escapeExemptTagName = null;
                        this.textEscapingMode = null;
                      }
                      type = HtmlTokenType.TAGBEGIN;
                      // Don't process content as attributes if we're inside
                      // an escape exempt block.
                      inTag = !this.inEscapeExemptBlock;
                      state = State.DONE;
                      break charloop;
                    }
                    break;
                  case SLASH:
                    if (Character.isLetter(ch)) {
                      state = State.TAGNAME;
                    } else {
                      if ('<' == ch) {
                        type = HtmlTokenType.TEXT;
                      } else {
                        ++end;
                      }
                      break charloop;
                    }
                    break;
                  case BANG:
                    if ('-' == ch) {
                      state = State.BANG_DASH;
                    } else {
                      state = State.DIRECTIVE;
                    }
                    break;
                  case BANG_DASH:
                    if ('-' == ch) {
                      state = State.COMMENT;
                    } else {
                      state = State.DIRECTIVE;
                    }
                    break;
                  case COMMENT:
                    if ('-' == ch) {
                      state = State.COMMENT_DASH;
                    }
                    break;
                  case COMMENT_DASH:
                    state = ('-' == ch)
                        ? State.COMMENT_DASH_DASH
                        : State.COMMENT_DASH;
                    break;
                  case COMMENT_DASH_DASH:
                    if ('>' == ch) {
                      state = State.DONE;
                      type = HtmlTokenType.COMMENT;
                    } else if ('-' == ch) {
                      state = State.COMMENT_DASH_DASH;
                    } else {
                      state = State.COMMENT_DASH;
                    }
                    break;
                  case DIRECTIVE:
                    if ('>' == ch) {
                      type = HtmlTokenType.DIRECTIVE;
                      state = State.DONE;
                    }
                    break;
                  case BOGUS_COMMENT:
                    if ('>' == ch) {
                      type = HtmlTokenType.QMARKMETA;
                      state = State.DONE;
                    }
                    break;
                  case SERVER_CODE:
                    if ('%' == ch) {
                      state = State.SERVER_CODE_PCT;
                    }
                    break;
                  case SERVER_CODE_PCT:
                    if ('>' == ch) {
                      type = HtmlTokenType.SERVERCODE;
                      state = State.DONE;
                    } else if ('%' != ch) {
                      state = State.SERVER_CODE;
                    }
                    break;
                  case DONE:
                    throw new AssertionError(
                        "Unexpectedly DONE while lexing HTML token stream");
                }
                ++end;
                if (State.DONE == state) { break; }
              }
              if (end == limit) {
                switch (state) {
                  case DONE:
                    break;
                  case BOGUS_COMMENT:
                    type = HtmlTokenType.QMARKMETA;
                    break;
                  case COMMENT:
                  case COMMENT_DASH:
                  case COMMENT_DASH_DASH:
                    type = HtmlTokenType.COMMENT;
                    break;
                  case DIRECTIVE:
                  case SERVER_CODE:
                  case SERVER_CODE_PCT:
                    type = HtmlTokenType.SERVERCODE;
                    break;
                  case TAGNAME:
                    type = HtmlTokenType.TAGBEGIN;
                    break;
                  default:
                    type = HtmlTokenType.TEXT;
                    break;
                }
              }
            }
          }
        } else {
          type = null;
        }
      }
      if (null == type) {
        while (end < limit && '<' != input.charAt(end)) { ++end; }
        type = HtmlTokenType.TEXT;
      }
  
      offset = end;
      HtmlToken result = HtmlToken.instance(start, end, type);
      if (type != HtmlTokenType.IGNORABLE) { lastNonIgnorable = result; }
      return result;
    }