docs: def find_optimal_tokenization(self, tokens):
          """
          Similar to {@link #findOptimalAllocation(String[])}, but also includes the words not matching any longer n-gram
          in TokenTrie as singletons.
  
          :param tokens: tokens to tokenize
          :return: Optimal allocation of tokens to phrases, with non matching tokens as singletons.
          """
          token_ranges = self.find_optimal_allocation(tokens)
          tokenized_sentence = []
  
          set_index = 0
          for token in token_ranges:
              while set_index < token.get_start_index():
                  tokenized_sentence.append(tokens[set_index])
                  set_index += 1
              tokenized_sentence.append(" ".join(token.get_token_sequence()))
              set_index = token.get_end_index() + 1
  
          while set_index < len(tokens):
              tokenized_sentence.append(tokens[set_index])
              set_index += 1
  
          return tokenized_sentence