code: public static List<Token> tokenize(String warpRawText, EvaluatorCompiler compiler) throws ExpressionCompileException {
      ArrayList<Token> tokens = new ArrayList<Token>();
  
      //simple state machine to iterate the text and break it up into chunks
      char[] characters = warpRawText.toCharArray();
  
      StringBuilder token = new StringBuilder();
      TokenizerState state = TokenizerState.READING_TEXT;
      for (int i = 0; i < characters.length; i++) {
  
        //test for start of an expression
        if (TokenizerState.READING_TEXT.equals(state)) {
          if ('$' == characters[i]) {
            if ('{' == characters[i + 1]) {
              //YES it is the start of an expr, so close up the existing token & start a new one
              if (token.length() > 0) {
                tokens.add(CompiledToken.text(token.toString()));
                token = new StringBuilder();
              }
  
              state = TokenizerState.READING_EXPRESSION;
            }
          }
        }
  
        //test for end of an expr
        if (TokenizerState.READING_EXPRESSION.equals(state)) {
          if ('}' == characters[i]) {
            //YES it is the end of the expr, so close it up and start a new token
            token.append(characters[i]);
  
            tokens.add(CompiledToken.expression(token.toString(), compiler));
            token = new StringBuilder();
  
            state = TokenizerState.READING_TEXT;
            continue;   //dont add the trailing } to the new text field
          }
        }
  
        //add characters to the token normally
        token.append(characters[i]);
      }
  
      //should never be in reading expr mode at this point
      if (TokenizerState.READING_EXPRESSION.equals(state))
        throw new IllegalStateException("Error. Expression was not terminated properly: " + token.toString());
  
      //add last token read if it has any content (is always text)
      if (token.length() > 0)
        tokens.add(CompiledToken.text(token.toString()));
  
      // Pack list capacity to size (saves memory).
      tokens.trimToSize();
  
      return tokens;
    }
docs: tokenizes text into raw text chunks interspersed with expression chunks