code: function defaultTokenizer(chars, wordOptions) {
          var word = chars.join(""), result, tokenRanges = [];
  
          function createTokenRange(start, end, isWord) {
              tokenRanges.push( { start: start, end: end, isWord: isWord } );
          }
  
          // Match words and mark characters
          var lastWordEnd = 0, wordStart, wordEnd;
          while ( (result = wordOptions.wordRegex.exec(word)) ) {
              wordStart = result.index;
              wordEnd = wordStart + result[0].length;
  
              // Create token for non-word characters preceding this word
              if (wordStart > lastWordEnd) {
                  createTokenRange(lastWordEnd, wordStart, false);
              }
  
              // Get trailing space characters for word
              if (wordOptions.includeTrailingSpace) {
                  while ( nonLineBreakWhiteSpaceRegex.test(chars[wordEnd]) ) {
                      ++wordEnd;
                  }
              }
              createTokenRange(wordStart, wordEnd, true);
              lastWordEnd = wordEnd;
          }
  
          // Create token for trailing non-word characters, if any exist
          if (lastWordEnd < chars.length) {
              createTokenRange(lastWordEnd, chars.length, false);
          }
  
          return tokenRanges;
      }
docs: /*---------------------------------------------------------------------------------------------------------------- This function must create word and non-word tokens for the whole of the text supplied to it