(self, langs: Set[str], min_len: int = 25)
return data
Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.
Args : langs min_len : Minimum text length in * chars * for a given example to be included .
def load(self, langs: Set[str], min_len: int = 25) -> List[Tuple[str, str]]:
        """
        Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
        """
        data = []
        match_regex = r"ud-(train|test|dev)\.txt"
        for fpath in tio.get_filepaths(
            self.data_dir, match_regex=match_regex, recursive=True
        ):
            fname = pathlib.Path(fpath).name
            lang, _ = fname.split("_", maxsplit=1)
            if lang not in langs:
                continue

            with open(fpath, mode="rt") as f:
                text = f.read()
            if "\n" in text:
                data.extend(
                    (text_segment, lang)
                    for text_segment in re.split(r"\n+", text)
                    if len(text_segment) >= min_len
                )
            else:
                data.extend(
                    (text_segment, lang)
                    for text_segment in _randomly_segment_text(text, (50, 1000))
                    if len(text_segment) >= min_len
                )
        LOGGER.info("loaded TatoebaDataset data:\n%s ...", data[:3])
        return data
def load ( self , langs : Set [ str ] , min_len : int = 25 ) -> List [ Tuple [ str , str ] ] : data = [ ] match_regex = r"ud-(train|test|dev)\.txt" for fpath in tio . get_filepaths ( self . data_dir , match_regex = match_regex , recursive = True ) : fname = pathlib . Path ( fpath ) . name lang , _ = fname . split ( "_" , maxsplit = 1 ) if lang not in langs : continue with open ( fpath , mode = "rt" ) as f : text = f . read ( ) if "\n" in text : data . extend ( ( text_segment , lang ) for text_segment in re . split ( r"\n+" , text ) if len ( text_segment ) >= min_len ) else : data . extend ( ( text_segment , lang ) for text_segment in _randomly_segment_text ( text , ( 50 , 1000 ) ) if len ( text_segment ) >= min_len ) LOGGER . info ( "loaded TatoebaDataset data:\n%s ..." , data [ : 3 ] ) return data