(
        self,
        iso_lang_map: Dict[str, str],
        min_len: int = 25,
    )
return data
Args:
            iso_lang_map
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
Args:
            iso_lang_map
            min_len: Minimum text length in *chars* for a given example to be included.
Args : iso_lang_map min_len : Minimum text length in * chars * for a given example to be included .
def load(
        self,
        iso_lang_map: Dict[str, str],
        min_len: int = 25,
    ) -> List[Tuple[str, str]]:
        """
        Args:
            iso_lang_map
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
        """
        data = []
        # we'll combine train/test from individual datasets
        # and instead split on the full, aggregated dataset
        for subset in ("train", "test"):
            text_lines = tio.read_text(
                self.data_dir.joinpath(f"x_{subset}.txt"), lines=True
            )
            lang_lines = tio.read_text(
                self.data_dir.joinpath(f"y_{subset}.txt"), lines=True
            )
            texts = (line.strip() for line in text_lines)
            langs = (line.strip() for line in lang_lines)
            data.extend(
                (text, iso_lang_map[lang])
                for text, lang in zip(texts, langs)
                if lang in iso_lang_map
                and itertoolz.count(char for char in text if char.isalnum()) >= min_len
            )
        LOGGER.info("loaded Wili2018Dataset data:\n%s ...", data[:3])
        return data
def load ( self , iso_lang_map : Dict [ str , str ] , min_len : int = 25 , ) -> List [ Tuple [ str , str ] ] : data = [ ] # we'll combine train/test from individual datasets # and instead split on the full, aggregated dataset for subset in ( "train" , "test" ) : text_lines = tio . read_text ( self . data_dir . joinpath ( f"x_{subset}.txt" ) , lines = True ) lang_lines = tio . read_text ( self . data_dir . joinpath ( f"y_{subset}.txt" ) , lines = True ) texts = ( line . strip ( ) for line in text_lines ) langs = ( line . strip ( ) for line in lang_lines ) data . extend ( ( text , iso_lang_map [ lang ] ) for text , lang in zip ( texts , langs ) if lang in iso_lang_map and itertoolz . count ( char for char in text if char . isalnum ( ) ) >= min_len ) LOGGER . info ( "loaded Wili2018Dataset data:\n%s ..." , data [ : 3 ] ) return data