(train_dir, vocab_dir, vocab_size=5000)
return 0
根据训练数据构建词汇表并存为 txt 文件
    :param train_dir: 训练数据路径
    :param vocab_dir: 词汇表存储路径
    :param vocab_size: 词汇表大小
    :return:
根据训练数据构建词汇表并存为 txt 文件
    :param train_dir: 训练数据路径
    :param vocab_dir: 词汇表存储路径
    :param vocab_size: 词汇表大小
    :return:
根据训练数据构建词汇表并存为 txt 文件 : param train_dir : 训练数据路径 : param vocab_dir : 词汇表存储路径 : param vocab_size : 词汇表大小 : return :
def build_vocab(train_dir, vocab_dir, vocab_size=5000):
    """
    根据训练数据构建词汇表并存为 txt 文件
    :param train_dir: 训练数据路径
    :param vocab_dir: 词汇表存储路径
    :param vocab_size: 词汇表大小
    :return:
    """
    data_train, _ = read_file(train_dir)

    all_data = []
    # 将字符串转为单个字符的list
    for content in data_train:
        for word in content:
            if word.strip():
                all_data.append(word)

    counter = collections.Counter(all_data)
    counter_pairs = counter.most_common(vocab_size - 2)
    words, _ = list(zip(*counter_pairs))
    words = ['<UNK>'] + list(words)
    words = ['<PAD>'] + list(words)

    with open(vocab_dir, "a") as f:
        f.write('\n'.join(words) + "\n")

    return 0
def build_vocab ( train_dir , vocab_dir , vocab_size = 5000 ) : data_train , _ = read_file ( train_dir ) all_data = [ ] # 将字符串转为单个字符的list for content in data_train : for word in content : if word . strip ( ) : all_data . append ( word ) counter = collections . Counter ( all_data ) counter_pairs = counter . most_common ( vocab_size - 2 ) words , _ = list ( zip ( * counter_pairs ) ) words = [ '<UNK>' ] + list ( words ) words = [ '<PAD>' ] + list ( words ) with open ( vocab_dir , "a" ) as f : f . write ( '\n' . join ( words ) + "\n" ) return 0