public static function tokenize($sentence, $options = array("HMM" => true))
    {
        $seg_list = self::cut($sentence, false, array("HMM" => $options["HMM"]));
        $tokenize_list = [];
        $start = 0;
        $end = 0;
        foreach ($seg_list as $seg) {
            $end = $start+mb_strlen($seg, 'UTF-8');
            $tokenize = [
                'word' => $seg,
                'start' => $start,
                'end' => $end
            ];
            $start = $end;
            array_push($tokenize_list, $tokenize);
        }
        return $tokenize_list;
    }
public static function tokenize($sentence, $options = array("HMM" => true))
    {
        $seg_list = self::cut($sentence, false, array("HMM" => $options["HMM"]));
        $tokenize_list = [];
        $start = 0;
        $end = 0;
        foreach ($seg_list as $seg) {
            $end = $start+mb_strlen($seg, 'UTF-8');
            $tokenize = [
                'word' => $seg,
                'start' => $start,
                'end' => $end
            ];
            $start = $end;
            array_push($tokenize_list, $tokenize);
        }
        return $tokenize_list;
    }
public static function tokenize ( $ sentence , $ options = array ( "HMM" => true ) ) { $ seg_list = self :: cut ( $ sentence , false , array ( "HMM" => $ options [ "HMM" ] ) ) ; $ tokenize_list = [ ] ; $ start = 0 ; $ end = 0 ; foreach ( $ seg_list as $ seg ) { $ end = $ start + mb_strlen ( $ seg , 'UTF-8' ) ; $ tokenize = [ 'word' => $ seg , 'start' => $ start , 'end' => $ end ] ; $ start = $ end ; array_push ( $ tokenize_list , $ tokenize ) ; } return $ tokenize_list ; }
Static method tokenize

@param string $sentence

@return array
Static method tokenize