code: function next_lextab_token(lexer) {
  
    var token;
  
    if(lexer.eos()) {
      return undefined;
    }
  
    // skip leading whitespace or comments...
    lexer.skip_filler();
  
    var previouslyPeeked = lexer.getPeekedToken();
    if(previouslyPeeked) {
      return previouslyPeeked;
    }
  
    lexer.mark_token_start();
    trace(lexer.message_src_loc("", lexer, {file:false}));
  
    // try and match all token categories except punctuation (and
    // the default handler).  Note that single character punctuation
    // should take a back seat to longer symbols e.g. '...' should
    // match before '.'.
    token = match_in_lextabs(lexer, {omit: ['punctuation','default']});
    if(!token) {
      // okay now try again matching punctuation characters.
      token = match_in_lextabs(lexer, {include: ['punctuation']});
    }
    if(!token) {
      // ok go ahead and try any default handler(s)
      token = match_in_lextabs(lexer, {include: ['default']});
    }
  
    // we expect they will use an explicit default lextab entry, but JIC:
    if(!token) {
      trace('their lextab has no default handler - defaulting to next word');
      token = lexer.next_word_token();
      if(token) {
        token.category = 'symbol';
      }
    }
  
    return token;
  }
docs: Get the next token under the current source position according
  to the lextab entries of the current dialects.