code: @Override
      public final boolean incrementToken() throws IOException
      {
          clearAttributes();
          int posIncr = 1;
          if(tokensList.size()>0)
          {
              this.restoreState(tokensList.remove());
              return true;
          }
          while (true)
          {
              int tokenType = scanner.getNextToken();
  
              if (tokenType == TweetTokenizerImpl.YYEOF)
              {
                  return false;
              }
  
              if (scanner.yylength() <= maxTokenLength)
              {
                  posIncrAtt.setPositionIncrement(posIncr);
                  scanner.getText(termAtt);
                  final int start = scanner.yychar();
                  offsetAtt.setOffset(correctOffset(start), correctOffset(start
                          + termAtt.length()));
                  // This 'if' should be removed in the next release. For now, it
                  // converts
                  // invalid acronyms to HOST. When removed, only the 'else' part
                  // should
                  // remain.
                  if (tokenType == TweetTokenizerImpl.ACRONYM_DEP)
                  {
                      if (replaceInvalidAcronym)
                      {
                          typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[TweetTokenizerImpl.HOST]);
                          termAtt.setLength(termAtt.length() - 1); // remove
                                                                           // extra
                                                                           // '.'
                      }
                      else
                      {
                          typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[TweetTokenizerImpl.ACRONYM]);
                      }
                  }
                  else
                  {
                      typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[tokenType]);
                  }
                  if(tokensList.size()>0)
                      this.restoreState(tokensList.remove());
                  /**
                   * End
                   */
                  return true;
              }
              else
                  // When we skip a too-long term, we still increment the
                  // position increment
                  posIncr++;
          }
      }
docs: /*
  (non-Javadoc)
  @see org.apache.lucene.analysis.TokenStream#next()