public function tokenize($str)
    {
        $str = array($str);
        foreach ($this->patterns as $p) {
            if (!is_array($p)) $p = array($p);
            if (count($p)==1) { // split pattern
                $this->split($str, $p[0]);
            } elseif (is_int($p[1])) { // match pattern
                $this->match($str, $p[0], $p[1]);
            } else { // replace pattern
                $this->replace($str, $p[0], $p[1]);
            }
        }

        return $str;
    }
public function tokenize($str)
    {
        $str = array($str);
        foreach ($this->patterns as $p) {
            if (!is_array($p)) $p = array($p);
            if (count($p)==1) { // split pattern
                $this->split($str, $p[0]);
            } elseif (is_int($p[1])) { // match pattern
                $this->match($str, $p[0], $p[1]);
            } else { // replace pattern
                $this->replace($str, $p[0], $p[1]);
            }
        }

        return $str;
    }
public function tokenize ( $ str ) { $ str = array ( $ str ) ; foreach ( $ this -> patterns as $ p ) { if ( ! is_array ( $ p ) ) $ p = array ( $ p ) ; if ( count ( $ p ) == 1 ) { // split pattern $ this -> split ( $ str , $ p [ 0 ] ) ; } elseif ( is_int ( $ p [ 1 ] ) ) { // match pattern $ this -> match ( $ str , $ p [ 0 ] , $ p [ 1 ] ) ; } else { // replace pattern $ this -> replace ( $ str , $ p [ 0 ] , $ p [ 1 ] ) ; } } return $ str ; }
Iteratively run for each pattern. The tokens resulting from one pattern are
fed to the next as strings.

If the pattern is given alone, it is assumed that it is a pattern used
for splitting with preg_split.

If the pattern is given together with an integer then it is assumed to be
a pattern used with preg_match

If a pattern is given with a string it is assumed to be a transformation
pattern used with preg_replace

@param  string $str The string to be tokenized
@return array  The tokens
Iteratively run for each pattern . The tokens resulting from one pattern are fed to the next as strings .