(pre_train_emb_path, text_vocab_path, embedding_path)
Prepare embedding
Prepare embedding
Prepare embedding
def prepare_embedding(pre_train_emb_path, text_vocab_path, embedding_path):
  """Prepare embedding"""
  # pylint: disable=too-many-locals

  if os.path.isfile(embedding_path):
    os.remove(embedding_path)

  # load pre_train model
  logging.info("Loading embedding from {}".format(pre_train_emb_path))
  emb_data = open(pre_train_emb_path, encoding='utf-8', mode='r')
  emb_dict = {}
  emb_size = 0
  line_num = 0
  for line in emb_data.readlines():
    line_num += 1
    if line_num == 1:
      continue
    line = line.strip().split(' ')
    word = line[0]
    vector = [float(i) for i in line[1:]]
    emb_dict[word] = vector
    emb_size = len(vector)
  logging.info("Load {} vectors".format(line_num))
  # load text vocab
  vocabs = load_vocab_dict(text_vocab_path)

  # get embedding vector for words in vocab
  vocab_size = len(vocabs)
  emb_list = [[]] * vocab_size
  bound = 1
  count_exist = 0
  count_not_exist = 0
  word_id = 0

  for word in vocabs:
    try:
      word_vector = emb_dict[word]
    except Exception:  # pylint: disable=broad-except
      word_vector = None

    if word_vector is not None:
      emb_list[word_id] = word_vector
      count_exist += 1
    else:
      count_not_exist += 1
      emb_list[word_id] = np.random.uniform(-bound, bound, emb_size)
    word_id += 1

  emb_list = np.array(emb_list)
  logging.info("embedding exist : {}, embedding not exist : {}".format(
      count_exist, count_not_exist))
  logging.info("embedding exist dump to: {}".format(embedding_path))
  with open(embedding_path, mode='wb') as out_f:
    pickle.dump(emb_list, out_f)
def prepare_embedding ( pre_train_emb_path , text_vocab_path , embedding_path ) : # pylint: disable=too-many-locals if os . path . isfile ( embedding_path ) : os . remove ( embedding_path ) # load pre_train model logging . info ( "Loading embedding from {}" . format ( pre_train_emb_path ) ) emb_data = open ( pre_train_emb_path , encoding = 'utf-8' , mode = 'r' ) emb_dict = { } emb_size = 0 line_num = 0 for line in emb_data . readlines ( ) : line_num += 1 if line_num == 1 : continue line = line . strip ( ) . split ( ' ' ) word = line [ 0 ] vector = [ float ( i ) for i in line [ 1 : ] ] emb_dict [ word ] = vector emb_size = len ( vector ) logging . info ( "Load {} vectors" . format ( line_num ) ) # load text vocab vocabs = load_vocab_dict ( text_vocab_path ) # get embedding vector for words in vocab vocab_size = len ( vocabs ) emb_list = [ [ ] ] * vocab_size bound = 1 count_exist = 0 count_not_exist = 0 word_id = 0 for word in vocabs : try : word_vector = emb_dict [ word ] except Exception : # pylint: disable=broad-except word_vector = None if word_vector is not None : emb_list [ word_id ] = word_vector count_exist += 1 else : count_not_exist += 1 emb_list [ word_id ] = np . random . uniform ( - bound , bound , emb_size ) word_id += 1 emb_list = np . array ( emb_list ) logging . info ( "embedding exist : {}, embedding not exist : {}" . format ( count_exist , count_not_exist ) ) logging . info ( "embedding exist dump to: {}" . format ( embedding_path ) ) with open ( embedding_path , mode = 'wb' ) as out_f : pickle . dump ( emb_list , out_f )