(fname, max_vocab_size, cache=True)
return vocab, vocab_size
Get ordered dict of vocab from google word2vec
Get ordered dict of vocab from google word2vec
Get ordered dict of vocab from google word2vec
def get_w2v_vocab(fname, max_vocab_size, cache=True):
    """
    Get ordered dict of vocab from google word2vec
    """
    if cache:
        cache_fname = fname.split('.')[0] + ".vocab"

        if os.path.isfile(cache_fname):
            vocab, vocab_size = load_obj(cache_fname)
            neon_logger.display("Word2Vec vocab cached, size is: {}".format(vocab_size))
            return vocab, vocab_size

    with open(fname, 'rb') as f:
        header = f.readline()
        vocab_size, embed_dim = map(int, header.split())
        binary_len = np.dtype('float32').itemsize * embed_dim

        neon_logger.display("Word2Vec vocab size is: {}".format(vocab_size))
        vocab_size = min(max_vocab_size, vocab_size)
        neon_logger.display("Reducing vocab size to: {}".format(vocab_size))

        vocab = OrderedDict()

        for i, line in enumerate(range(vocab_size)):
            word = []
            while True:
                ch = f.read(1)
                if ch == b' ':
                    word = (b''.join(word)).decode('utf-8')
                    break
                if ch != b'\n':
                    word.append(ch)
            f.read(binary_len)
            vocab[word] = i

    if cache:
        save_obj((vocab, vocab_size), cache_fname)

    return vocab, vocab_size
def get_w2v_vocab ( fname , max_vocab_size , cache = True ) : if cache : cache_fname = fname . split ( '.' ) [ 0 ] + ".vocab" if os . path . isfile ( cache_fname ) : vocab , vocab_size = load_obj ( cache_fname ) neon_logger . display ( "Word2Vec vocab cached, size is: {}" . format ( vocab_size ) ) return vocab , vocab_size with open ( fname , 'rb' ) as f : header = f . readline ( ) vocab_size , embed_dim = map ( int , header . split ( ) ) binary_len = np . dtype ( 'float32' ) . itemsize * embed_dim neon_logger . display ( "Word2Vec vocab size is: {}" . format ( vocab_size ) ) vocab_size = min ( max_vocab_size , vocab_size ) neon_logger . display ( "Reducing vocab size to: {}" . format ( vocab_size ) ) vocab = OrderedDict ( ) for i , line in enumerate ( range ( vocab_size ) ) : word = [ ] while True : ch = f . read ( 1 ) if ch == b' ' : word = ( b'' . join ( word ) ) . decode ( 'utf-8' ) break if ch != b'\n' : word . append ( ch ) f . read ( binary_len ) vocab [ word ] = i if cache : save_obj ( ( vocab , vocab_size ) , cache_fname ) return vocab , vocab_size