public function tokenize($str)
    {
        // split the string in tokens and create documents to be
        // classified
        $tokens = $this->tok->tokenize($str);
        $docs = array();
        foreach ($tokens as $offset=>$tok) {
            $docs[] = new WordDocument($tokens,$offset,5);
        }

        // classify each token as an EOW or O
        $tags = array();
        foreach ($docs as $doc) {
            $tags[] = $this->classifier->classify(self::$classSet, $doc);
        }

        // merge O and EOW into real tokens
        $realtokens = array();
        $currentToken = array();
        foreach ($tokens as $offset=>$tok) {
            $currentToken[] = $tok;
            if ($tags[$offset] == self::EOW) {
                $realtokens[] = implode($this->sep,$currentToken);
                $currentToken = array();
            }
        }

        // return real tokens
        return $realtokens;
    }
public function tokenize($str)
    {
        // split the string in tokens and create documents to be
        // classified
        $tokens = $this->tok->tokenize($str);
        $docs = array();
        foreach ($tokens as $offset=>$tok) {
            $docs[] = new WordDocument($tokens,$offset,5);
        }

        // classify each token as an EOW or O
        $tags = array();
        foreach ($docs as $doc) {
            $tags[] = $this->classifier->classify(self::$classSet, $doc);
        }

        // merge O and EOW into real tokens
        $realtokens = array();
        $currentToken = array();
        foreach ($tokens as $offset=>$tok) {
            $currentToken[] = $tok;
            if ($tags[$offset] == self::EOW) {
                $realtokens[] = implode($this->sep,$currentToken);
                $currentToken = array();
            }
        }

        // return real tokens
        return $realtokens;
    }
public function tokenize ( $ str ) { // split the string in tokens and create documents to be // classified $ tokens = $ this -> tok -> tokenize ( $ str ) ; $ docs = array ( ) ; foreach ( $ tokens as $ offset => $ tok ) { $ docs [ ] = new WordDocument ( $ tokens , $ offset , 5 ) ; } // classify each token as an EOW or O $ tags = array ( ) ; foreach ( $ docs as $ doc ) { $ tags [ ] = $ this -> classifier -> classify ( self :: $ classSet , $ doc ) ; } // merge O and EOW into real tokens $ realtokens = array ( ) ; $ currentToken = array ( ) ; foreach ( $ tokens as $ offset => $ tok ) { $ currentToken [ ] = $ tok ; if ( $ tags [ $ offset ] == self :: EOW ) { $ realtokens [ ] = implode ( $ this -> sep , $ currentToken ) ; $ currentToken = array ( ) ; } } // return real tokens return $ realtokens ; }
Tokenize the string.

1. Break up the string in tokens using the initial tokenizer
2. Classify each token if it is an EOW
3. For each token that is not an EOW add it to the next EOW token using a separator

@param  string $str The character sequence to be broken in tokens
@return array  The token array
Tokenize the string .