(self, text, top_k=5, with_weight=False,
                          func_word_num=1, stop_word_num=0, 
                          max_phrase_len=25,
                          topic_theta=0.5, allow_pos_weight=True,
                          stricted_pos=True, allow_length_weight=True,
                          allow_topic_weight=True,
                          without_person_name=False,
                          without_location_name=False,
                          remove_phrases_list=None,
                          remove_words_list=None,
                          specified_words=dict(), bias=None)
抽取一篇文本的关键短语
        :param text: utf-8 编码中文文本
        :param top_k: 选取多少个关键短语返回，默认为 5，若为 -1 返回所有短语
        :param with_weight: 指定返回关键短语是否需要短语权重
        :param func_word_num: 允许短语中出现的虚词个数，stricted_pos 为 True 时无效
        :param stop_word_num: 允许短语中出现的停用词个数，stricted_pos 为 True 时无效
        :param max_phrase_len: 允许短语的最长长度，默认为 25 个字符
        :param topic_theta: 主题权重的权重调节因子，默认0.5，范围（0~无穷）
        :param stricted_pos: (bool) 为 True 时仅允许名词短语出现
        :param allow_pos_weight: (bool) 考虑词性权重，即某些词性组合的短语首尾更倾向成为关键短语
        :param allow_length_weight: (bool) 考虑词性权重，即 token 长度为 2~5 的短语倾向成为关键短语
        :param allow_topic_weight: (bool) 考虑主题突出度，它有助于过滤与主题无关的短语（如日期等）
        :param without_person_name: (bool) 决定是否剔除短语中的人名
        :param without_location_name: (bool) 决定是否剔除短语中的地名
        :param remove_phrases_list: (list) 将某些不想要的短语剔除，使其不出现在最终结果中
        :param remove_words_list: (list) 将某些不想要的词剔除，使包含该词的短语不出现在最终结果中
        :param specified_words: (dict) 行业名词:词频，若不为空，则仅返回包含该词的短语
        :param bias: (int|float) 若指定 specified_words，则可选择定义权重增加值
        :return: 关键短语及其权重
抽取一篇文本的关键短语
        :param text: utf-8 编码中文文本
        :param top_k: 选取多少个关键短语返回，默认为 5，若为 -1 返回所有短语
        :param with_weight: 指定返回关键短语是否需要短语权重
        :param func_word_num: 允许短语中出现的虚词个数，stricted_pos 为 True 时无效
        :param stop_word_num: 允许短语中出现的停用词个数，stricted_pos 为 True 时无效
        :param max_phrase_len: 允许短语的最长长度，默认为 25 个字符
        :param topic_theta: 主题权重的权重调节因子，默认0.5，范围（0~无穷）
        :param stricted_pos: (bool) 为 True 时仅允许名词短语出现
        :param allow_pos_weight: (bool) 考虑词性权重，即某些词性组合的短语首尾更倾向成为关键短语
        :param allow_length_weight: (bool) 考虑词性权重，即 token 长度为 2~5 的短语倾向成为关键短语
        :param allow_topic_weight: (bool) 考虑主题突出度，它有助于过滤与主题无关的短语（如日期等）
        :param without_person_name: (bool) 决定是否剔除短语中的人名
        :param without_location_name: (bool) 决定是否剔除短语中的地名
        :param remove_phrases_list: (list) 将某些不想要的短语剔除，使其不出现在最终结果中
        :param remove_words_list: (list) 将某些不想要的词剔除，使包含该词的短语不出现在最终结果中
        :param specified_words: (dict) 行业名词:词频，若不为空，则仅返回包含该词的短语
        :param bias: (int|float) 若指定 specified_words，则可选择定义权重增加值
        :return: 关键短语及其权重
抽取一篇文本的关键短语 : param text : utf - 8 编码中文文本 : param top_k : 选取多少个关键短语返回，默认为 5，若为 - 1 返回所有短语 : param with_weight : 指定返回关键短语是否需要短语权重 : param func_word_num : 允许短语中出现的虚词个数，stricted_pos 为 True 时无效 : param stop_word_num : 允许短语中出现的停用词个数，stricted_pos 为 True 时无效 : param max_phrase_len : 允许短语的最长长度，默认为 25 个字符 : param topic_theta : 主题权重的权重调节因子，默认0 . 5，范围（0~无穷） : param stricted_pos : ( bool ) 为 True 时仅允许名词短语出现 : param allow_pos_weight : ( bool ) 考虑词性权重，即某些词性组合的短语首尾更倾向成为关键短语 : param allow_length_weight : ( bool ) 考虑词性权重，即 token 长度为 2~5 的短语倾向成为关键短语 : param allow_topic_weight : ( bool ) 考虑主题突出度，它有助于过滤与主题无关的短语（如日期等） : param without_person_name : ( bool ) 决定是否剔除短语中的人名 : param without_location_name : ( bool ) 决定是否剔除短语中的地名 : param remove_phrases_list : ( list ) 将某些不想要的短语剔除，使其不出现在最终结果中 : param remove_words_list : ( list ) 将某些不想要的词剔除，使包含该词的短语不出现在最终结果中 : param specified_words : ( dict ) 行业名词 : 词频，若不为空，则仅返回包含该词的短语 : param bias : ( int|float ) 若指定 specified_words，则可选择定义权重增加值 : return : 关键短语及其权重
def extract_keyphrase(self, text, top_k=5, with_weight=False,
                          func_word_num=1, stop_word_num=0, 
                          max_phrase_len=25,
                          topic_theta=0.5, allow_pos_weight=True,
                          stricted_pos=True, allow_length_weight=True,
                          allow_topic_weight=True,
                          without_person_name=False,
                          without_location_name=False,
                          remove_phrases_list=None,
                          remove_words_list=None,
                          specified_words=dict(), bias=None):
        """
        抽取一篇文本的关键短语
        :param text: utf-8 编码中文文本
        :param top_k: 选取多少个关键短语返回，默认为 5，若为 -1 返回所有短语
        :param with_weight: 指定返回关键短语是否需要短语权重
        :param func_word_num: 允许短语中出现的虚词个数，stricted_pos 为 True 时无效
        :param stop_word_num: 允许短语中出现的停用词个数，stricted_pos 为 True 时无效
        :param max_phrase_len: 允许短语的最长长度，默认为 25 个字符
        :param topic_theta: 主题权重的权重调节因子，默认0.5，范围（0~无穷）
        :param stricted_pos: (bool) 为 True 时仅允许名词短语出现
        :param allow_pos_weight: (bool) 考虑词性权重，即某些词性组合的短语首尾更倾向成为关键短语
        :param allow_length_weight: (bool) 考虑词性权重，即 token 长度为 2~5 的短语倾向成为关键短语
        :param allow_topic_weight: (bool) 考虑主题突出度，它有助于过滤与主题无关的短语（如日期等）
        :param without_person_name: (bool) 决定是否剔除短语中的人名
        :param without_location_name: (bool) 决定是否剔除短语中的地名
        :param remove_phrases_list: (list) 将某些不想要的短语剔除，使其不出现在最终结果中
        :param remove_words_list: (list) 将某些不想要的词剔除，使包含该词的短语不出现在最终结果中
        :param specified_words: (dict) 行业名词:词频，若不为空，则仅返回包含该词的短语
        :param bias: (int|float) 若指定 specified_words，则可选择定义权重增加值
        :return: 关键短语及其权重
        """ 
        try:
            # 配置参数
            if without_location_name:
                if 'ns' in self.stricted_pos_name:
                    self.stricted_pos_name.remove('ns')
                if 'ns' in self.pos_name:
                    self.pos_name.remove('ns')
            else:
                if 'ns' not in self.stricted_pos_name:
                    self.stricted_pos_name.append('ns')
                if 'ns' not in self.pos_name:
                    self.pos_name.append('ns')

            if without_person_name:
                if 'nr' in self.stricted_pos_name:
                    self.stricted_pos_name.remove('nr')
                if 'nr' in self.pos_name:
                    self.pos_name.remove('nr')
            else:
                if 'nr' not in self.stricted_pos_name:
                    self.stricted_pos_name.append('nr')
                if 'nr' not in self.pos_name:
                    self.pos_name.append('nr')

            # step0: 清洗文本，去除杂质
            text = self._preprocessing_text(text)

            # step1: 分句，使用北大的分词器 pkuseg 做分词和词性标注
            sentences_list = self._split_sentences(text)
            sentences_segs_list = list()
            counter_segs_list = list()
            for sen in sentences_list:
                sen_segs = self.seg.cut(sen)
                sentences_segs_list.append(sen_segs)
                counter_segs_list.extend(sen_segs)

            # step2: 计算词频
            total_length = len(counter_segs_list)
            freq_dict = dict()
            for word_pos in counter_segs_list:
                word, pos = word_pos
                if word in freq_dict:
                    freq_dict[word][1] += 1
                else:
                    freq_dict.update({word: [pos, 1]})

            # step3: 计算每一个词的权重
            sentences_segs_weights_list = list()
            for sen, sen_segs in zip(sentences_list, sentences_segs_list):
                sen_segs_weights = list()
                for word_pos in sen_segs:
                    word, pos = word_pos
                    if pos in self.pos_name:  # 虚词权重为 0
                        if word in self.stop_words:  # 停用词权重为 0
                            weight = 0.0
                        else:
                            if word in specified_words:  # 为词计算权重
                                if bias is None:
                                    weight = freq_dict[word][1] * self.idf_dict.get(
                                        word, self.median_idf) / total_length + 1 / specified_words[word]
                                else:
                                    weight = freq_dict[word][1] * self.idf_dict.get(
                                        word, self.median_idf) / total_length + bias
                            else:
                                weight = freq_dict[word][1] * self.idf_dict.get(
                                    word, self.median_idf) / total_length
                    else:
                        weight = 0.0
                    sen_segs_weights.append(weight)
                sentences_segs_weights_list.append(sen_segs_weights)

            # step4: 通过一定规则，找到候选短语集合，以及其权重
            candidate_phrases_dict = dict()
            for sen_segs, sen_segs_weights in zip(
                sentences_segs_list, sentences_segs_weights_list):
                sen_length = len(sen_segs)

                for n in range(1, sen_length + 1):  # n-grams
                    for i in range(0, sen_length - n + 1):
                        candidate_phrase = sen_segs[i: i + n]

                        # 由于 pkuseg 的缺陷，日期被识别为 n 而非 t，故删除日期
                        res = self.extra_date_ptn.match(candidate_phrase[-1][0])
                        if res is not None:
                            continue

                        # 找短语过程中需要进行过滤，分为严格、宽松规则
                        if not stricted_pos:  
                            rule_flag = self._loose_candidate_phrases_rules(
                                candidate_phrase, func_word_num=func_word_num,
                                max_phrase_len=max_phrase_len,  
                                stop_word_num=stop_word_num)
                        else:
                            rule_flag = self._stricted_candidate_phrases_rules(
                                candidate_phrase, max_phrase_len=max_phrase_len)
                        if not rule_flag:
                            continue

                        # 由于 pkuseg 的缺陷，会把一些杂质符号识别为 n、v、adj，故须删除
                        redundent_flag = False
                        for item in candidate_phrase:
                            matched = self.redundent_strict_pattern.search(item[0])
                            if matched is not None:
                                redundent_flag = True
                                break
                            matched = self.redundent_loose_pattern.search(item[0])

                            if matched is not None and matched.group() == item[0]:
                                redundent_flag = True
                                break
                        if redundent_flag:
                            continue
                            
                        # 如果短语中包含了某些不想要的词，则跳过
                        if remove_words_list is not None:
                            unwanted_phrase_flag = False
                            for item in candidate_phrase:
                                if item[0] in remove_words_list:
                                    unwanted_phrase_flag = True
                                    break
                            if unwanted_phrase_flag:
                                continue

                        # 如果短语中没有一个 token 存在于指定词汇中，则跳过
                        if specified_words != dict():
                            with_specified_words_flag = False
                            for item in candidate_phrase:
                                if item[0] in specified_words:
                                    with_specified_words_flag = True
                                    break
                            if not with_specified_words_flag:
                                continue

                        # 条件六：短语的权重需要乘上'词性权重'
                        if allow_pos_weight:
                            start_end_pos = None
                            if len(candidate_phrase) == 1:
                                start_end_pos = candidate_phrase[0][1]
                            elif len(candidate_phrase) >= 2:
                                start_end_pos = candidate_phrase[0][1] + '|' + candidate_phrase[-1][1]
                            pos_weight = self.pos_combine_weights_dict.get(start_end_pos, 1.0)
                        else:
                            pos_weight = 1.0

                        # 条件七：短语的权重需要乘上 '长度权重'
                        if allow_length_weight:
                            length_weight = self.phrases_length_control_dict.get(
                                len(sen_segs_weights[i: i + n]), 
                                self.phrases_length_control_none)
                        else:
                            length_weight = 1.0

                        # 条件八：短语的权重需要加上`主题突出度权重`
                        if allow_topic_weight:
                            topic_weight = 0.0
                            for item in candidate_phrase:
                                topic_weight += self.topic_prominence_dict.get(
                                    item[0], self.unk_topic_prominence_value)
                            topic_weight = topic_weight / len(candidate_phrase)
                        else:
                            topic_weight = 0.0

                        candidate_phrase_weight = sum(sen_segs_weights[i: i + n])
                        candidate_phrase_weight *= length_weight * pos_weight
                        candidate_phrase_weight += topic_weight * topic_theta

                        candidate_phrase_string = ''.join([tup[0] for tup in candidate_phrase])
                        if remove_phrases_list is not None:
                            if candidate_phrase_string in remove_phrases_list:
                                continue
                        if candidate_phrase_string not in candidate_phrases_dict:
                            candidate_phrases_dict.update(
                                {candidate_phrase_string: [candidate_phrase, 
                                                           candidate_phrase_weight]})

            # step5: 将 overlaping 过量的短语进行去重过滤
            # 尝试了依据权重高低，将较短的短语替代重复了的较长的短语，但效果不好，故删去
            candidate_phrases_list = sorted(
                candidate_phrases_dict.items(), 
                key=lambda item: len(item[1][0]), reverse=True)

            de_duplication_candidate_phrases_list = list()
            for item in candidate_phrases_list:
                sim_ratio = self._mmr_similarity(
                    item, de_duplication_candidate_phrases_list)
                if sim_ratio != 1:
                    item[1][1] = (1 - sim_ratio) * item[1][1]
                    de_duplication_candidate_phrases_list.append(item)

            # step6: 按重要程度进行排序，选取 top_k 个
            candidate_phrases_list = sorted(de_duplication_candidate_phrases_list, 
                                            key=lambda item: item[1][1], reverse=True)

            if with_weight:
                if top_k != -1:
                    final_res = [(item[0], item[1][1]) for item in candidate_phrases_list[:top_k]
                                 if item[1][1] > 0]
                else:
                    final_res = [(item[0], item[1][1]) for item in candidate_phrases_list
                                 if item[1][1] > 0]
            else:
                if top_k != -1:
                    final_res = [item[0] for item in candidate_phrases_list[:top_k]
                                 if item[1][1] > 0]
                else:
                    final_res = [item[0] for item in candidate_phrases_list
                                 if item[1][1] > 0]
            return final_res

        except Exception as e:
            print('the text is not legal. \n{}'.format(e))
            return []
def extract_keyphrase ( self , text , top_k = 5 , with_weight = False , func_word_num = 1 , stop_word_num = 0 , max_phrase_len = 25 , topic_theta = 0.5 , allow_pos_weight = True , stricted_pos = True , allow_length_weight = True , allow_topic_weight = True , without_person_name = False , without_location_name = False , remove_phrases_list = None , remove_words_list = None , specified_words = dict ( ) , bias = None ) : try : # 配置参数 if without_location_name : if 'ns' in self . stricted_pos_name : self . stricted_pos_name . remove ( 'ns' ) if 'ns' in self . pos_name : self . pos_name . remove ( 'ns' ) else : if 'ns' not in self . stricted_pos_name : self . stricted_pos_name . append ( 'ns' ) if 'ns' not in self . pos_name : self . pos_name . append ( 'ns' ) if without_person_name : if 'nr' in self . stricted_pos_name : self . stricted_pos_name . remove ( 'nr' ) if 'nr' in self . pos_name : self . pos_name . remove ( 'nr' ) else : if 'nr' not in self . stricted_pos_name : self . stricted_pos_name . append ( 'nr' ) if 'nr' not in self . pos_name : self . pos_name . append ( 'nr' ) # step0: 清洗文本，去除杂质 text = self . _preprocessing_text ( text ) # step1: 分句，使用北大的分词器 pkuseg 做分词和词性标注 sentences_list = self . _split_sentences ( text ) sentences_segs_list = list ( ) counter_segs_list = list ( ) for sen in sentences_list : sen_segs = self . seg . cut ( sen ) sentences_segs_list . append ( sen_segs ) counter_segs_list . extend ( sen_segs ) # step2: 计算词频 total_length = len ( counter_segs_list ) freq_dict = dict ( ) for word_pos in counter_segs_list : word , pos = word_pos if word in freq_dict : freq_dict [ word ] [ 1 ] += 1 else : freq_dict . update ( { word : [ pos , 1 ] } ) # step3: 计算每一个词的权重 sentences_segs_weights_list = list ( ) for sen , sen_segs in zip ( sentences_list , sentences_segs_list ) : sen_segs_weights = list ( ) for word_pos in sen_segs : word , pos = word_pos if pos in self . pos_name : # 虚词权重为 0 if word in self . stop_words : # 停用词权重为 0 weight = 0.0 else : if word in specified_words : # 为词计算权重 if bias is None : weight = freq_dict [ word ] [ 1 ] * self . idf_dict . get ( word , self . median_idf ) / total_length + 1 / specified_words [ word ] else : weight = freq_dict [ word ] [ 1 ] * self . idf_dict . get ( word , self . median_idf ) / total_length + bias else : weight = freq_dict [ word ] [ 1 ] * self . idf_dict . get ( word , self . median_idf ) / total_length else : weight = 0.0 sen_segs_weights . append ( weight ) sentences_segs_weights_list . append ( sen_segs_weights ) # step4: 通过一定规则，找到候选短语集合，以及其权重 candidate_phrases_dict = dict ( ) for sen_segs , sen_segs_weights in zip ( sentences_segs_list , sentences_segs_weights_list ) : sen_length = len ( sen_segs ) for n in range ( 1 , sen_length + 1 ) : # n-grams for i in range ( 0 , sen_length - n + 1 ) : candidate_phrase = sen_segs [ i : i + n ] # 由于 pkuseg 的缺陷，日期被识别为 n 而非 t，故删除日期 res = self . extra_date_ptn . match ( candidate_phrase [ - 1 ] [ 0 ] ) if res is not None : continue # 找短语过程中需要进行过滤，分为严格、宽松规则 if not stricted_pos : rule_flag = self . _loose_candidate_phrases_rules ( candidate_phrase , func_word_num = func_word_num , max_phrase_len = max_phrase_len , stop_word_num = stop_word_num ) else : rule_flag = self . _stricted_candidate_phrases_rules ( candidate_phrase , max_phrase_len = max_phrase_len ) if not rule_flag : continue # 由于 pkuseg 的缺陷，会把一些杂质符号识别为 n、v、adj，故须删除 redundent_flag = False for item in candidate_phrase : matched = self . redundent_strict_pattern . search ( item [ 0 ] ) if matched is not None : redundent_flag = True break matched = self . redundent_loose_pattern . search ( item [ 0 ] ) if matched is not None and matched . group ( ) == item [ 0 ] : redundent_flag = True break if redundent_flag : continue # 如果短语中包含了某些不想要的词，则跳过 if remove_words_list is not None : unwanted_phrase_flag = False for item in candidate_phrase : if item [ 0 ] in remove_words_list : unwanted_phrase_flag = True break if unwanted_phrase_flag : continue # 如果短语中没有一个 token 存在于指定词汇中，则跳过 if specified_words != dict ( ) : with_specified_words_flag = False for item in candidate_phrase : if item [ 0 ] in specified_words : with_specified_words_flag = True break if not with_specified_words_flag : continue # 条件六：短语的权重需要乘上'词性权重' if allow_pos_weight : start_end_pos = None if len ( candidate_phrase ) == 1 : start_end_pos = candidate_phrase [ 0 ] [ 1 ] elif len ( candidate_phrase ) >= 2 : start_end_pos = candidate_phrase [ 0 ] [ 1 ] + '|' + candidate_phrase [ - 1 ] [ 1 ] pos_weight = self . pos_combine_weights_dict . get ( start_end_pos , 1.0 ) else : pos_weight = 1.0 # 条件七：短语的权重需要乘上 '长度权重' if allow_length_weight : length_weight = self . phrases_length_control_dict . get ( len ( sen_segs_weights [ i : i + n ] ) , self . phrases_length_control_none ) else : length_weight = 1.0 # 条件八：短语的权重需要加上`主题突出度权重` if allow_topic_weight : topic_weight = 0.0 for item in candidate_phrase : topic_weight += self . topic_prominence_dict . get ( item [ 0 ] , self . unk_topic_prominence_value ) topic_weight = topic_weight / len ( candidate_phrase ) else : topic_weight = 0.0 candidate_phrase_weight = sum ( sen_segs_weights [ i : i + n ] ) candidate_phrase_weight *= length_weight * pos_weight candidate_phrase_weight += topic_weight * topic_theta candidate_phrase_string = '' . join ( [ tup [ 0 ] for tup in candidate_phrase ] ) if remove_phrases_list is not None : if candidate_phrase_string in remove_phrases_list : continue if candidate_phrase_string not in candidate_phrases_dict : candidate_phrases_dict . update ( { candidate_phrase_string : [ candidate_phrase , candidate_phrase_weight ] } ) # step5: 将 overlaping 过量的短语进行去重过滤 # 尝试了依据权重高低，将较短的短语替代重复了的较长的短语，但效果不好，故删去 candidate_phrases_list = sorted ( candidate_phrases_dict . items ( ) , key = lambda item : len ( item [ 1 ] [ 0 ] ) , reverse = True ) de_duplication_candidate_phrases_list = list ( ) for item in candidate_phrases_list : sim_ratio = self . _mmr_similarity ( item , de_duplication_candidate_phrases_list ) if sim_ratio != 1 : item [ 1 ] [ 1 ] = ( 1 - sim_ratio ) * item [ 1 ] [ 1 ] de_duplication_candidate_phrases_list . append ( item ) # step6: 按重要程度进行排序，选取 top_k 个 candidate_phrases_list = sorted ( de_duplication_candidate_phrases_list , key = lambda item : item [ 1 ] [ 1 ] , reverse = True ) if with_weight : if top_k != - 1 : final_res = [ ( item [ 0 ] , item [ 1 ] [ 1 ] ) for item in candidate_phrases_list [ : top_k ] if item [ 1 ] [ 1 ] > 0 ] else : final_res = [ ( item [ 0 ] , item [ 1 ] [ 1 ] ) for item in candidate_phrases_list if item [ 1 ] [ 1 ] > 0 ] else : if top_k != - 1 : final_res = [ item [ 0 ] for item in candidate_phrases_list [ : top_k ] if item [ 1 ] [ 1 ] > 0 ] else : final_res = [ item [ 0 ] for item in candidate_phrases_list if item [ 1 ] [ 1 ] > 0 ] return final_res except Exception as e : print ( 'the text is not legal. \n{}' . format ( e ) ) return [ ]