code: private String[] getTokens(String line) {
  
      // these are fine because they do not affect offsets
      line = line.trim();
      line = doubleSpaces.matcher(line).replaceAll(" ");
      // remove non printable stuff
      line = asciiHex.matcher(line).replaceAll(" ");
      line = generalBlankPunctuation.matcher(line).replaceAll(" ");
  
      // separate question and exclamation marks
      line = qexc.matcher(line).replaceAll(" $1 ");
      // separate dash if before or after space
      line = spaceDashSpace.matcher(line).replaceAll(" $1 ");
      // tokenize everything but these characters [^\p{Alnum}s.'`,-?!/]
      line = specials.matcher(line).replaceAll(" $1 ");
  
      // do not separate multidots
      line = generateMultidots(line);
  
      // separate "," except if within numbers (1,200)
      line = noDigitComma.matcher(line).replaceAll("$1 $2");
      line = commaNoDigit.matcher(line).replaceAll("$1 $2");
      // separate pre and post digit
      line = digitCommaNoDigit.matcher(line).replaceAll("$1 $2 $3");
      line = noDigitCommaDigit.matcher(line).replaceAll("$1 $2 $3");
  
      // contractions it's, l'agila, c'est, don't
      line = treatContractions(line);
      // exceptions for period tokenization
      line = this.nonBreaker.TokenizerNonBreaker(line);
  
      // restore multidots
      line = restoreMultidots(line);
      // urls
      // TODO normalize URLs after tokenization for offsets
      line = detokenizeURLs(line);
      line = beginLink.matcher(line).replaceAll("$1://");
      line = endLink.matcher(line).replaceAll("$1$2");
  
      // these are fine because they do not affect offsets
      line = line.trim();
      line = doubleSpaces.matcher(line).replaceAll(" ");
      line = detokenParagraphs.matcher(line).replaceAll("$1$2");
  
      if (DEBUG) {
        System.out.println("->Tokens:" + line);
      }
      final String[] tokens = line.split(" ");
      return tokens;
    }
docs: Actual tokenization function.
  
  @param line
  the sentence to be tokenized
  @return an array containing the tokens for the sentence