docs: public Object next() throws java.io.IOException {
      int zzInput;
      int zzAction;
  
      // cached fields:
      int zzCurrentPosL;
      int zzMarkedPosL;
      int zzEndReadL = zzEndRead;
      char [] zzBufferL = zzBuffer;
      char [] zzCMapL = ZZ_CMAP;
  
      int [] zzTransL = ZZ_TRANS;
      int [] zzRowMapL = ZZ_ROWMAP;
      int [] zzAttrL = ZZ_ATTRIBUTE;
  
      while (true) {
        zzMarkedPosL = zzMarkedPos;
  
        yychar+= zzMarkedPosL-zzStartRead;
  
        zzAction = -1;
  
        zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
    
        zzState = ZZ_LEXSTATE[zzLexicalState];
  
  
        zzForAction: {
          while (true) {
      
            if (zzCurrentPosL < zzEndReadL)
              zzInput = zzBufferL[zzCurrentPosL++];
            else if (zzAtEOF) {
              zzInput = YYEOF;
              break zzForAction;
            }
            else {
              // store back cached positions
              zzCurrentPos  = zzCurrentPosL;
              zzMarkedPos   = zzMarkedPosL;
              boolean eof = zzRefill();
              // get translated positions and possibly new buffer
              zzCurrentPosL  = zzCurrentPos;
              zzMarkedPosL   = zzMarkedPos;
              zzBufferL      = zzBuffer;
              zzEndReadL     = zzEndRead;
              if (eof) {
                zzInput = YYEOF;
                break zzForAction;
              }
              else {
                zzInput = zzBufferL[zzCurrentPosL++];
              }
            }
            int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
            if (zzNext == -1) break zzForAction;
            zzState = zzNext;
  
            int zzAttributes = zzAttrL[zzState];
            if ( (zzAttributes & 1) == 1 ) {
              zzAction = zzState;
              zzMarkedPosL = zzCurrentPosL;
              if ( (zzAttributes & 8) == 8 ) break zzForAction;
            }
  
          }
        }
  
        // store back cached position
        zzMarkedPos = zzMarkedPosL;
  
        switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
          case 30: 
            // lookahead expression with fixed base length
            zzMarkedPos = zzStartRead + 1;
            { /* invert quote - often but not always right */
  		  return handleQuotes(yytext(), true);
            }
          case 43: break;
          case 8: 
            { if (ptb3Dashes) {
                              return getNext(ptbmdash, yytext()); }
                            else {
                              return getNext();
                            }
            }
          case 44: break;
          case 19: 
            { if (escapeForwardSlashAsterisk) {
                      return getNext(delimit(yytext(), '*'), yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 45: break;
          case 15: 
            { return handleQuotes(yytext(), false);
            }
          case 46: break;
          case 29: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return handleQuotes(yytext(), false);
            }
          case 47: break;
          case 12: 
            { if (yylength() >= 3 && yylength() <= 4 && ptb3Dashes) {
  	            return getNext(ptbmdash, yytext());
                    } else {
                      return getNext();
  		  }
            }
          case 48: break;
          case 38: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return getNormalizedAmpNext();
            }
          case 49: break;
          case 28: 
            { return getNormalizedAmpNext();
            }
          case 50: break;
          case 4: 
            { return getNext();
            }
          case 51: break;
          case 21: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return getNext();
            }
          case 52: break;
          case 33: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 11;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 12;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { return getNext();
            }
          case 53: break;
          case 34: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 13;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 12;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { return getNext();
            }
          case 54: break;
          case 40: 
            { String txt = yytext();
                            if (escapeForwardSlashAsterisk) {
                              txt = delimit(txt, '/');
                            }
                            return getNext(txt, yytext());
            }
          case 55: break;
          case 23: 
            { final String origTxt = yytext();
  			  String txt = origTxt;
         			  if (normalizeSpace) {
  			    txt = txt.replaceAll(" ", "\u00A0"); // change to non-breaking space
  			  }
                            return getNext(txt, origTxt);
            }
          case 56: break;
          case 6: 
            { if (normalizeOtherBrackets) {
                      return getNext(closeparen, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 57: break;
          case 26: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 3;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 4;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { final String origTxt = yytext();
                            String tmp = removeSoftHyphens(origTxt);
                            if (americanize) {
                              tmp = Americanize.americanize(tmp);
                            }
                            return getNext(tmp, origTxt);
            }
          case 58: break;
          case 16: 
            { if (normalizeOtherBrackets) {
                      return getNext(closebrace, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 59: break;
          case 24: 
            { if (escapeForwardSlashAsterisk) {
                              return getNext(delimit(yytext(), '/'), yytext());
                            } else {
                              return getNext();
                            }
            }
          case 60: break;
          case 18: 
            { if (normalizeParentheses) {
                      return getNext(closeparen, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 61: break;
          case 36: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 10;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { String s;
                            if (strictTreebank3) {
                              yypushback(1); // return a period for next time
                              s = yytext();
                            } else {
                              s = yytext();
                              yypushback(1); // return a period for next time
                            }
  	                  return getNext(s, yytext());
            }
          case 62: break;
          case 39: 
            { yypushback(3) ; return getNext();
            }
          case 63: break;
          case 22: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return handleQuotes(yytext(), true);
            }
          case 64: break;
          case 41: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 7;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 8;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { String s;
                            // try to fight around an apparent jflex bug where it gets a space at the token
                            // end by getting wrong the length of the trailing context.
                            while (yylength() > 0) {
  			    char last = yycharat(yylength()-1);
  			    if (last == ' ' || last == '\t') {
                                yypushback(1);
                              } else {
                                break;
                              }
                            }
                            if (strictTreebank3) {
                              yypushback(1); // return a period for next time
                              s = yytext();
                            } else {
                              s = yytext();
                              yypushback(1); // return a period for next time
                            }
                            return getNext(s, yytext());
            }
          case 65: break;
          case 2: 
            { if (normalizeOtherBrackets) {
                      return getNext(openparen, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 66: break;
          case 32: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 5;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                if ((zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
                zzInput = zzBufferL[zzFPos++];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1 && (zzAttrL[zzFState] & 1) == 1) { zzFinL[zzFPos] = true; } 
  
              zzFState = 6;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = zzBufferL[--zzFPos];
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { final String txt = yytext();
                            return getNext(removeSoftHyphens(txt),
                                           txt);
            }
          case 67: break;
          case 20: 
            { if (normalizeOtherBrackets) {
                      return getNext(openbrace, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 68: break;
          case 17: 
            { if (normalizeParentheses) {
                      return getNext(openparen, yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 69: break;
          case 11: 
            { return handleEllipsis(yytext());
            }
          case 70: break;
          case 13: 
            { return normalizeFractions(yytext());
            }
          case 71: break;
          case 14: 
            { if (normalizeCurrency) {
                              return getNext(normalizeCurrency(yytext()), yytext()); }
                            else {
                              return getNext();
                            }
            }
          case 72: break;
          case 10: 
            { if (invertible) {
                       prevWordAfter.append(yytext());
                    }
            }
          case 73: break;
          case 3: 
            { if (escapeForwardSlashAsterisk) {
                      return getNext(delimit(yytext(), '/'), yytext()); }
                    else {
                      return getNext();
                    }
            }
          case 74: break;
          case 37: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return getNext(removeSoftHyphens(yytext()),
                                                 yytext());
            }
          case 75: break;
          case 25: 
            { return getNext(removeSoftHyphens(yytext()), yytext());
            }
          case 76: break;
          case 31: 
            { String txt = yytext();
                    if (escapeForwardSlashAsterisk) {
                      txt = delimit(txt, '/');
                    }
                    if (normalizeSpace) {
                      txt = txt.replaceAll(" ", "\u00A0"); // change to non-breaking space
                    }
                    return getNext(txt, yytext());
            }
          case 77: break;
          case 7: 
            { final String origTxt = yytext();
  			  String tmp = removeSoftHyphens(origTxt);
                            if (americanize) {
                              tmp = Americanize.americanize(tmp);
                            }
                            return getNext(tmp, origTxt);
            }
          case 78: break;
          case 42: 
            { String txt = yytext();
  			  if (normalizeSpace) {
  			    txt = txt.replaceAll(" ", "\u00A0"); // change to non-breaking space
  			  }
  			  if (normalizeParentheses) {
  			    txt = txt.replaceAll("\\(", openparen);
  			    txt = txt.replaceAll("\\)", closeparen);
  			  }
  			  return getNext(txt, yytext());
            }
          case 79: break;
          case 9: 
            { return getNext(removeSoftHyphens(yytext()),
                                           yytext());
            }
          case 80: break;
          case 27: 
            // lookahead expression with fixed lookahead length
            yypushback(1);
            { return getNext(removeSoftHyphens(yytext()),
                                           yytext());
            }
          case 81: break;
          case 1: 
            { String str = yytext();
            int first = str.charAt(0);
            String msg = String.format("Untokenizable: %s (U+%s, decimal: %s)", yytext(), Integer.toHexString(first).toUpperCase(), Integer.toString(first));
            switch (untokenizable) {
              case NONE_DELETE:
                if (invertible) {
                  prevWordAfter.append(str);
                }
                break;
              case FIRST_DELETE:
                if (invertible) {
                  prevWordAfter.append(str);
                }
                if ( ! this.seenUntokenizableCharacter) {
                  LOGGER.warning(msg);
                  this.seenUntokenizableCharacter = true;
                }
                break;
              case ALL_DELETE:
                if (invertible) {
                  prevWordAfter.append(str);
                }
                LOGGER.warning(msg);
                this.seenUntokenizableCharacter = true;
                break;
              case NONE_KEEP:
                return getNext();
              case FIRST_KEEP:
                if ( ! this.seenUntokenizableCharacter) {
                  LOGGER.warning(msg);
                  this.seenUntokenizableCharacter = true;
                }
                return getNext();
              case ALL_KEEP:
                LOGGER.warning(msg);
                this.seenUntokenizableCharacter = true;
                return getNext();
            }
            }
          case 82: break;
          case 5: 
            { if (tokenizeNLs) {
                        return getNext(NEWLINE_TOKEN, yytext()); // js: for tokenizing carriage returns
                    } else if (invertible) {
                        prevWordAfter.append(yytext());
                    }
            }
          case 83: break;
          case 35: 
            { String txt = yytext();
                            if (escapeForwardSlashAsterisk) {
                              txt = delimit(txt, '/');
                              txt = delimit(txt, '*');
                            }
                            return getNext(txt, yytext());
            }
          case 84: break;
          default: 
            if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
              zzAtEOF = true;
                {
                  if (invertible) { 
              prevWordAfter.append(yytext()); 
              String str = prevWordAfter.toString();
              prevWordAfter.setLength(0);
              prevWord.set(AfterAnnotation.class, str);
            }
            return null;
                }
            } 
            else {
              zzScanError(ZZ_NO_MATCH);
            }
        }
      }
    }