(questions, contexts)
return vocab, counter
Build vocabulary sorted by global word frequency, or consider frequencies in questions first,
    which is controlled by `args.sort_all`.
Build vocabulary sorted by global word frequency, or consider frequencies in questions first,
    which is controlled by `args.sort_all`.
Build vocabulary sorted by global word frequency or consider frequencies in questions first which is controlled by args . sort_all .
def build_vocab(questions, contexts):
    '''
    Build vocabulary sorted by global word frequency, or consider frequencies in questions first,
    which is controlled by `args.sort_all`.
    '''
    if args.sort_all:
        counter = collections.Counter(w for doc in questions + contexts for w in doc)
        vocab = sorted([t for t in counter if t in wv_vocab], key=counter.get, reverse=True)
    else:
        counter_q = collections.Counter(w for doc in questions for w in doc)
        counter_c = collections.Counter(w for doc in contexts for w in doc)
        counter = counter_c + counter_q
        vocab = sorted([t for t in counter_q if t in wv_vocab], key=counter_q.get, reverse=True)
        vocab += sorted([t for t in counter_c.keys() - counter_q.keys() if t in wv_vocab],
                        key=counter.get, reverse=True)
    total = sum(counter.values())
    matched = sum(counter[t] for t in vocab)
    log.info('vocab coverage {1}/{0} | OOV occurrence {2}/{3} ({4:.4f}%)'.format(
        len(counter), len(vocab), (total - matched), total, (total - matched) / total * 100))
    vocab.insert(0, "<PAD>")
    vocab.insert(1, "<UNK>")
    return vocab, counter
def build_vocab ( questions , contexts ) : if args . sort_all : counter = collections . Counter ( w for doc in questions + contexts for w in doc ) vocab = sorted ( [ t for t in counter if t in wv_vocab ] , key = counter . get , reverse = True ) else : counter_q = collections . Counter ( w for doc in questions for w in doc ) counter_c = collections . Counter ( w for doc in contexts for w in doc ) counter = counter_c + counter_q vocab = sorted ( [ t for t in counter_q if t in wv_vocab ] , key = counter_q . get , reverse = True ) vocab += sorted ( [ t for t in counter_c . keys ( ) - counter_q . keys ( ) if t in wv_vocab ] , key = counter . get , reverse = True ) total = sum ( counter . values ( ) ) matched = sum ( counter [ t ] for t in vocab ) log . info ( 'vocab coverage {1}/{0} | OOV occurrence {2}/{3} ({4:.4f}%)' . format ( len ( counter ) , len ( vocab ) , ( total - matched ) , total , ( total - matched ) / total * 100 ) ) vocab . insert ( 0 , "<PAD>" ) vocab . insert ( 1 , "<UNK>" ) return vocab , counter