code: func (t TreebankWordTokenizer) Tokenize(text string) []string {
  	for substitution, r := range startingQuotes {
  		text = r.ReplaceAllString(text, substitution)
  	}
  
  	for substitution, r := range startingQuotes2 {
  		text = r.ReplaceAllString(text, substitution)
  	}
  
  	for substitution, r := range punctuation {
  		text = r.ReplaceAllString(text, substitution)
  	}
  
  	for _, r := range punctuation2 {
  		text = r.ReplaceAllString(text, " $1 ")
  	}
  
  	for substitution, r := range brackets {
  		text = r.ReplaceAllString(text, substitution)
  	}
  
  	text = " " + text + " "
  
  	for substitution, r := range endingQuotes {
  		text = r.ReplaceAllString(text, substitution)
  	}
  
  	for _, r := range endingQuotes2 {
  		text = r.ReplaceAllString(text, "$1 $2 ")
  	}
  
  	for _, r := range contractions {
  		text = r.ReplaceAllString(text, " $1 $2 ")
  	}
  
  	text = newlines.ReplaceAllString(text, " ")
  	text = strings.TrimSpace(spaces.ReplaceAllString(text, " "))
  	return strings.Split(text, " ")
  }
docs: // Tokenize splits a sentence into a slice of words.
  //
  // This tokenizer performs the following steps: (1) split on contractions (e.g.,
  // "don't" -> [do n't]), (2) split on non-terminating punctuation, (3) split on
  // single quotes when followed by whitespace, and (4) split on periods that
  // appear at the end of lines.
  //
  // NOTE: As mentioned above, this function expects a sentence (not raw text) as
  // input.