protected function tokenize($string)
    {
        $string = preg_replace('(\\r\\n|\\r)', "\n", $string);
        $tokens = array();
        $line = 1;

        while (strlen($string)) {
            foreach ($this->tokenSpecifications as $token => $regexp) {
                if (!preg_match($regexp, $string, $matches)) {
                    continue;
                }

                // Remove matched string
                $string = substr($string, strlen($matches[0]));
                $line += substr_count($matches[0], "\n");

                // Append token to list
                $tokens[] = array(
                    'type' => $token,
                    'data' => $this->filterValues($matches),
                );

                // Continue with outer loop
                continue 2;
            }

            throw new RuntimeException("Parse error in line $line: " . substr($string, 0, 100));
        }

        return $tokens;
    }
protected function tokenize($string)
    {
        $string = preg_replace('(\\r\\n|\\r)', "\n", $string);
        $tokens = array();
        $line = 1;

        while (strlen($string)) {
            foreach ($this->tokenSpecifications as $token => $regexp) {
                if (!preg_match($regexp, $string, $matches)) {
                    continue;
                }

                // Remove matched string
                $string = substr($string, strlen($matches[0]));
                $line += substr_count($matches[0], "\n");

                // Append token to list
                $tokens[] = array(
                    'type' => $token,
                    'data' => $this->filterValues($matches),
                );

                // Continue with outer loop
                continue 2;
            }

            throw new RuntimeException("Parse error in line $line: " . substr($string, 0, 100));
        }

        return $tokens;
    }
protected function tokenize ( $ string ) { $ string = preg_replace ( '(\\r\\n|\\r)' , "\n" , $ string ) ; $ tokens = array ( ) ; $ line = 1 ; while ( strlen ( $ string ) ) { foreach ( $ this -> tokenSpecifications as $ token => $ regexp ) { if ( ! preg_match ( $ regexp , $ string , $ matches ) ) { continue ; } // Remove matched string $ string = substr ( $ string , strlen ( $ matches [ 0 ] ) ) ; $ line += substr_count ( $ matches [ 0 ] , "\n" ) ; // Append token to list $ tokens [ ] = array ( 'type' => $ token , 'data' => $ this -> filterValues ( $ matches ) , ) ; // Continue with outer loop continue 2 ; } throw new RuntimeException ( "Parse error in line $line: " . substr ( $ string , 0 , 100 ) ) ; } return $ tokens ; }
Tokenize transformation input file.

Returns an array of tokens

@param string $string

@return array
Tokenize transformation input file .