(input_file,out_file,max_len=52)
序列标注的训练数据构建
    :param input_file: 训练文件
    :param out_file: 输出文件
    :param max_len: 最大长度
    :return:
序列标注的训练数据构建
    :param input_file: 训练文件
    :param out_file: 输出文件
    :param max_len: 最大长度
    :return:
序列标注的训练数据构建 : param input_file : 训练文件 : param out_file : 输出文件 : param max_len : 最大长度 : return :
def get_input_bert(input_file,out_file,max_len=52):
    '''
    序列标注的训练数据构建
    :param input_file: 训练文件
    :param out_file: 输出文件
    :param max_len: 最大长度
    :return:
    '''
    token_dict = get_token_dict()
    tokenizer = Tokenizer(token_dict)
    bio_dict = get_bio_dict()
    inputs = {'ids': [], 'seg': [], 'labels': []}
    with open(input_file, 'r') as f:
        for line in f:
            temDict = json.loads(line)
            text = temDict['text']
            mention_data = temDict['mention_data']
            label = get_bio_list(text, mention_data)
            label = [bio_dict['TAG']] + label + [bio_dict['TAG']]
            indices, segments = get_ids_seg(text, tokenizer)
            assert len(indices)==len(label)
            inputs['ids'].append(seq_padding(indices, max_len))
            inputs['seg'].append(seq_padding(segments, max_len))
            inputs['labels'].append(seq_padding(label, max_len))

    for key in inputs:
        inputs[key] = np.array(inputs[key])
        print(key, inputs[key].shape)
        print(inputs[key][0])
    pd.to_pickle(inputs, out_file)
def get_input_bert ( input_file , out_file , max_len = 52 ) : token_dict = get_token_dict ( ) tokenizer = Tokenizer ( token_dict ) bio_dict = get_bio_dict ( ) inputs = { 'ids' : [ ] , 'seg' : [ ] , 'labels' : [ ] } with open ( input_file , 'r' ) as f : for line in f : temDict = json . loads ( line ) text = temDict [ 'text' ] mention_data = temDict [ 'mention_data' ] label = get_bio_list ( text , mention_data ) label = [ bio_dict [ 'TAG' ] ] + label + [ bio_dict [ 'TAG' ] ] indices , segments = get_ids_seg ( text , tokenizer ) assert len ( indices ) == len ( label ) inputs [ 'ids' ] . append ( seq_padding ( indices , max_len ) ) inputs [ 'seg' ] . append ( seq_padding ( segments , max_len ) ) inputs [ 'labels' ] . append ( seq_padding ( label , max_len ) ) for key in inputs : inputs [ key ] = np . array ( inputs [ key ] ) print ( key , inputs [ key ] . shape ) print ( inputs [ key ] [ 0 ] ) pd . to_pickle ( inputs , out_file )