(dirpath, iso_lang_map, min_len=25)
return ds
Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]

    References:
        https://zenodo.org/record/841984
Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*
Args : dirpath ( str ) iso_lang_map ( Dict [ str str ] ) min_len ( int ) : minimum text length in * chars *
def load_wili_data(dirpath, iso_lang_map, min_len=25):
    """
    Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]

    References:
        https://zenodo.org/record/841984
    """
    dirpath = textacy.utils.to_path(dirpath).resolve()
    ds = []
    for subset in ("train", "test"):
        text_lines = textacy.io.read_text(
            dirpath.joinpath("x_{}.txt".format(subset)), lines=True
        )
        lang_lines = textacy.io.read_text(
            dirpath.joinpath("y_{}.txt".format(subset)), lines=True
        )
        texts = (line.strip() for line in text_lines)
        langs = (line.strip() for line in lang_lines)
        langs_set = set(iso_lang_map.keys())
        ds.extend(
            (text, iso_lang_map[lang])
            for text, lang in zip(texts, langs)
            if lang in langs_set
            and itertoolz.count(char for char in text if char.isalnum()) >= min_len
        )
    return ds
def load_wili_data ( dirpath , iso_lang_map , min_len = 25 ) : dirpath = textacy . utils . to_path ( dirpath ) . resolve ( ) ds = [ ] for subset in ( "train" , "test" ) : text_lines = textacy . io . read_text ( dirpath . joinpath ( "x_{}.txt" . format ( subset ) ) , lines = True ) lang_lines = textacy . io . read_text ( dirpath . joinpath ( "y_{}.txt" . format ( subset ) ) , lines = True ) texts = ( line . strip ( ) for line in text_lines ) langs = ( line . strip ( ) for line in lang_lines ) langs_set = set ( iso_lang_map . keys ( ) ) ds . extend ( ( text , iso_lang_map [ lang ] ) for text , lang in zip ( texts , langs ) if lang in langs_set and itertoolz . count ( char for char in text if char . isalnum ( ) ) >= min_len ) return ds