code: func NewTokenizer(input io.RuneReader) *Tokenizer {
  	assert2(input != nil, "input must not be nil")
  	return &Tokenizer{
  		TokenStreamImpl: NewTokenStream(),
  		inputPending:    input,
  		Input:           ILLEGAL_STATE_READER,
  	}
  }
docs: /* Constructs a token stream processing the given input. */