public function tokenize($input)
    {
        // The list of tokens we'll eventually return
        $tokens = [];

        // The currentIndex indicates where we are inside the input string
        $currentIndex = 0;

        while ($currentIndex < strlen($input)) {
            // We try to match only what is after the currentIndex,
            // as the content before is already converted to tokens
            $token = $this->findMatchingToken(substr($input, $currentIndex));

            // If no tokens were matched, it means that the string has invalid tokens
            // for which we did not define a token definition
            if (!$token) {
                throw new UnknownTokenException(substr($input, $currentIndex));
            }

            // Add the matched token to our list of token
            $tokens[] = $token;

            // Increment the string index by the lenght of the matched token,
            // so we can now process the rest of the string.
            $currentIndex += $token->length();
        }

        return $tokens;
    }
public function tokenize($input)
    {
        // The list of tokens we'll eventually return
        $tokens = [];

        // The currentIndex indicates where we are inside the input string
        $currentIndex = 0;

        while ($currentIndex < strlen($input)) {
            // We try to match only what is after the currentIndex,
            // as the content before is already converted to tokens
            $token = $this->findMatchingToken(substr($input, $currentIndex));

            // If no tokens were matched, it means that the string has invalid tokens
            // for which we did not define a token definition
            if (!$token) {
                throw new UnknownTokenException(substr($input, $currentIndex));
            }

            // Add the matched token to our list of token
            $tokens[] = $token;

            // Increment the string index by the lenght of the matched token,
            // so we can now process the rest of the string.
            $currentIndex += $token->length();
        }

        return $tokens;
    }
public function tokenize ( $ input ) { // The list of tokens we'll eventually return $ tokens = [ ] ; // The currentIndex indicates where we are inside the input string $ currentIndex = 0 ; while ( $ currentIndex < strlen ( $ input ) ) { // We try to match only what is after the currentIndex, // as the content before is already converted to tokens $ token = $ this -> findMatchingToken ( substr ( $ input , $ currentIndex ) ) ; // If no tokens were matched, it means that the string has invalid tokens // for which we did not define a token definition if ( ! $ token ) { throw new UnknownTokenException ( substr ( $ input , $ currentIndex ) ) ; } // Add the matched token to our list of token $ tokens [ ] = $ token ; // Increment the string index by the lenght of the matched token, // so we can now process the rest of the string. $ currentIndex += $ token -> length ( ) ; } return $ tokens ; }
Convert an input string to a list of tokens.

Using the list of knowns tokens, sequentially match the input string to
known tokens. Note that the first matching token from the list is chosen,
so if there are tokens sharing parts of the pattern (e.g. `sin` and `sinh`),
care should be taken to add `sinh` before `sin`, otherwise the lexer will
never match a `sinh`.

@retval Token[] sequence of recognized tokens
that doesn't match any knwon token.
@param  string                $input  String to tokenize.
@throws UnknownTokenException throwns when encountering characters in the input string
Convert an input string to a list of tokens .