(vocab)
return stats, indices
Count frequency of all symbol pairs, and create index
Count frequency of all symbol pairs, and create index
Count frequency of all symbol pairs and create index
def get_pair_statistics(vocab):
    """Count frequency of all symbol pairs, and create index"""

    # data structure of pair frequencies
    stats = defaultdict(int)

    # index from pairs to words
    indices = defaultdict(lambda: defaultdict(int))

    for i, (word, freq) in enumerate(vocab):
        prev_char = word[0]
        for char in word[1:]:
            stats[prev_char, char] += freq
            indices[prev_char, char][i] += 1
            prev_char = char

    return stats, indices
def get_pair_statistics ( vocab ) : # data structure of pair frequencies stats = defaultdict ( int ) # index from pairs to words indices = defaultdict ( lambda : defaultdict ( int ) ) for i , ( word , freq ) in enumerate ( vocab ) : prev_char = word [ 0 ] for char in word [ 1 : ] : stats [ prev_char , char ] += freq indices [ prev_char , char ] [ i ] += 1 prev_char = char return stats , indices