(self, langs: Set[str], min_len: int = 25)
return data
Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.
Args : langs min_len : Minimum text length in * chars * for a given example to be included .
def load(self, langs: Set[str], min_len: int = 25) -> List[Tuple[str, str]]:
        """
        Args:
            langs
            min_len: Minimum text length in *chars* for a given example to be included.

        Returns:
            Sequence of (text, lang) examples.
        """
        data = []
        fstubs = [
            "dslcc3/train/task1-train.txt",
            "dslcc3/train/task1-dev.txt",
            "dslcc4/DSL-TRAIN.txt",
            "dslcc4/DSL-DEV.txt",
        ]
        for fstub in fstubs:
            filepath = self.data_dir.joinpath(fstub)
            lines = tio.read_text(filepath, mode="rt", encoding="utf-8", lines=True)
            for line in lines:
                if not line.strip():
                    continue
                try:
                    text, lang = line.split("\t")
                    if (
                        lang[:2] in langs
                        and itertoolz.count(c for c in text if c.isalnum()) >= min_len
                    ):
                        data.append((text, lang[:2]))
                except Exception:
                    LOGGER.debug("bad line in data")
                    pass
        data = sorted(set(data), key=operator.itemgetter(1))
        LOGGER.info("loaded DSLCCDataset data:\n%s ...", data[:3])
        return data
def load ( self , langs : Set [ str ] , min_len : int = 25 ) -> List [ Tuple [ str , str ] ] : data = [ ] fstubs = [ "dslcc3/train/task1-train.txt" , "dslcc3/train/task1-dev.txt" , "dslcc4/DSL-TRAIN.txt" , "dslcc4/DSL-DEV.txt" , ] for fstub in fstubs : filepath = self . data_dir . joinpath ( fstub ) lines = tio . read_text ( filepath , mode = "rt" , encoding = "utf-8" , lines = True ) for line in lines : if not line . strip ( ) : continue try : text , lang = line . split ( "\t" ) if ( lang [ : 2 ] in langs and itertoolz . count ( c for c in text if c . isalnum ( ) ) >= min_len ) : data . append ( ( text , lang [ : 2 ] ) ) except Exception : LOGGER . debug ( "bad line in data" ) pass data = sorted ( set ( data ) , key = operator . itemgetter ( 1 ) ) LOGGER . info ( "loaded DSLCCDataset data:\n%s ..." , data [ : 3 ] ) return data