parameters: (self, data_path)
return_statement: return data
docstring: :return: dict[language, list[embedding]]
docstring_summary: :return: dict[language, list[embedding]]
docstring_tokens: : return : dict [ language list [ embedding ]]
function: def load_data(self, data_path):
          """
          :return: dict[language, list[embedding]]
          """
          data = defaultdict(lambda: list())
  
          filenames = [f for f in os.listdir(data_path) if not f.startswith('.')]
  
          for file in filenames:
              file_no_gz = file[:-3] if file.endswith('.gz') else file
              split = file_no_gz.split('.')
              language = split[-2]
              vector_space = split[-1]
  
              if self.vector_space is None:
                  self.vector_space = vector_space
  
              if vector_space == self.vector_space:
                  file_path = os.path.join(data_path, file)
                  if not file.endswith('.gz'):
                      f = open(file_path, 'r')
                  else:
                      f = gzip.open(file_path, 'r')
                  try:
                      for line in f:
                          if len(data[language]) >= self.max_load:
                              break
                          line = line.strip().split()
                          if line:
                              embedding = np.asarray([float(s) for s in line], np.float32)
                              data[language].append(embedding)
                  finally:
                      f.close()
          return data
function_tokens: def load_data ( self , data_path ) : data = defaultdict ( lambda : list ( ) ) filenames = [ f for f in os . listdir ( data_path ) if not f . startswith ( '.' ) ] for file in filenames : file_no_gz = file [ : - 3 ] if file . endswith ( '.gz' ) else file split = file_no_gz . split ( '.' ) language = split [ - 2 ] vector_space = split [ - 1 ] if self . vector_space is None : self . vector_space = vector_space if vector_space == self . vector_space : file_path = os . path . join ( data_path , file ) if not file . endswith ( '.gz' ) : f = open ( file_path , 'r' ) else : f = gzip . open ( file_path , 'r' ) try : for line in f : if len ( data [ language ] ) >= self . max_load : break line = line . strip ( ) . split ( ) if line : embedding = np . asarray ( [ float ( s ) for s in line ] , np . float32 ) data [ language ] . append ( embedding ) finally : f . close ( ) return data