code: public String TokenizerNonBreaker(String line) {
  
      // these are fine because they do not affect offsets
      line = line.trim();
      line = RuleBasedTokenizer.doubleSpaces.matcher(line).replaceAll(" ");
      final StringBuilder sb = new StringBuilder();
      String tokenizedText = "";
      int i;
      final String[] words = line.split(" ");
  
      for (i = 0; i < words.length; i++) {
        final Matcher wordDotMatcher = wordDot.matcher(words[i]);
  
        // find anything non-whitespace finishing with a period
        if (wordDotMatcher.find()) {
          final String curWord = wordDotMatcher.replaceAll("$1");
  
          if (curWord.contains(".") && alphabetic.matcher(curWord).find()
              || curWord.matches("(" + this.NON_BREAKER + ")")
              || i < words.length - 1 && (startLower.matcher(words[i + 1]).find()
                  || startPunct.matcher(words[i + 1]).find())) {
            // do not tokenize if (word contains a period and is alphabetic) OR
            // word is a non breaker OR (word is a non breaker and next is
            // (lowercase or starts with punctuation that is end of sentence
            // marker))
          } else if (curWord.matches(NON_BREAKER_DIGITS) && i < words.length - 1
              && startDigit.matcher(words[i + 1]).find()) {
            // do not tokenize if word is a nonbreaker digit AND next word starts
            // with a digit
          } else {
            words[i] = curWord + " .";
          }
        }
        sb.append(words[i]).append(" ");
        tokenizedText = sb.toString();
      }
      return tokenizedText;
    }
docs: It decides when periods do not need to be tokenized.
  
  @param line
  the sentence to be tokenized
  @return line