(dirpath, iso_lang_map, min_len=25)
return ds
Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]

    References:
        http://wortschatz.uni-leipzig.de/en/download

    TODO: figure out a good way of downloading leipzig data programatically?
    TODO: only use news data, maybe?
Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*
Args : dirpath ( str ) iso_lang_map ( Dict [ str str ] ) min_len ( int ) : minimum text length in * chars *
def load_leipzig_data(dirpath, iso_lang_map, min_len=25):
    """
    Args:
        dirpath (str)
        iso_lang_map (Dict[str, str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]

    References:
        http://wortschatz.uni-leipzig.de/en/download

    TODO: figure out a good way of downloading leipzig data programatically?
    TODO: only use news data, maybe?
    """
    ds = []
    for filepath in sorted(textacy.io.get_filepaths(dirpath, match_regex=r"\.tar\.gz$")):
        fname = os.path.basename(filepath)
        lang = fname.split("_")[0]
        try:
            lang = iso_lang_map[lang]
        except KeyError:
            continue
        with tarfile.open(filepath, mode="r") as tf:
            for member in tf:
                if re.search(r".*?-sentences\.txt$", member.name):
                    with tf.extractfile(member) as f:
                        for line in f:
                            idx, text = line.decode("utf-8").split(sep="\t", maxsplit=1)
                            text = textacy.preprocessing.normalize_whitespace(text)
                            if len(text) >= min_len:
                                ds.append((text.strip(), lang))
    return ds
def load_leipzig_data ( dirpath , iso_lang_map , min_len = 25 ) : ds = [ ] for filepath in sorted ( textacy . io . get_filepaths ( dirpath , match_regex = r"\.tar\.gz$" ) ) : fname = os . path . basename ( filepath ) lang = fname . split ( "_" ) [ 0 ] try : lang = iso_lang_map [ lang ] except KeyError : continue with tarfile . open ( filepath , mode = "r" ) as tf : for member in tf : if re . search ( r".*?-sentences\.txt$" , member . name ) : with tf . extractfile ( member ) as f : for line in f : idx , text = line . decode ( "utf-8" ) . split ( sep = "\t" , maxsplit = 1 ) text = textacy . preprocessing . normalize_whitespace ( text ) if len ( text ) >= min_len : ds . append ( ( text . strip ( ) , lang ) ) return ds