(self, data_path)
return data
:return: dict[language, list[embedding]]
:return: dict[language, list[embedding]]
: return : dict [ language list [ embedding ]]
def load_data(self, data_path):
        """
        :return: dict[language, list[embedding]]
        """
        data = defaultdict(lambda: list())

        filenames = [f for f in os.listdir(data_path) if not f.startswith('.')]

        for file in filenames:
            file_no_gz = file[:-3] if file.endswith('.gz') else file
            split = file_no_gz.split('.')
            language = split[-2]
            vector_space = split[-1]

            if self.vector_space is None:
                self.vector_space = vector_space

            if vector_space == self.vector_space:
                file_path = os.path.join(data_path, file)
                if not file.endswith('.gz'):
                    f = open(file_path, 'r')
                else:
                    f = gzip.open(file_path, 'r')
                try:
                    for line in f:
                        if len(data[language]) >= self.max_load:
                            break
                        line = line.strip().split()
                        if line:
                            embedding = np.asarray([float(s) for s in line], np.float32)
                            data[language].append(embedding)
                finally:
                    f.close()
        return data
def load_data ( self , data_path ) : data = defaultdict ( lambda : list ( ) ) filenames = [ f for f in os . listdir ( data_path ) if not f . startswith ( '.' ) ] for file in filenames : file_no_gz = file [ : - 3 ] if file . endswith ( '.gz' ) else file split = file_no_gz . split ( '.' ) language = split [ - 2 ] vector_space = split [ - 1 ] if self . vector_space is None : self . vector_space = vector_space if vector_space == self . vector_space : file_path = os . path . join ( data_path , file ) if not file . endswith ( '.gz' ) : f = open ( file_path , 'r' ) else : f = gzip . open ( file_path , 'r' ) try : for line in f : if len ( data [ language ] ) >= self . max_load : break line = line . strip ( ) . split ( ) if line : embedding = np . asarray ( [ float ( s ) for s in line ] , np . float32 ) data [ language ] . append ( embedding ) finally : f . close ( ) return data