(
    root_dirpath: pathlib.Path, force: bool, min_len: int, min_obs: int
)
return agg_data
Download, load, and aggregate datasets.
Download, load, and aggregate datasets.
Download load and aggregate datasets .
def load_and_agg_data(
    root_dirpath: pathlib.Path, force: bool, min_len: int, min_obs: int
) -> List[Tuple[str, str]]:
    """Download, load, and aggregate datasets."""
    iso_lang_resource = textacy.lang_id_._datasets.IsoLangResource(
        root_dirpath.joinpath("iso-639")
    )
    iso_lang_resource.download(force=force)
    iso_lang_map = iso_lang_resource.load(exclude={"sh"})  # TODO: why exclude sh?
    valid_langs = set(iso_lang_map.values())

    udhr = textacy.datasets.UDHR(root_dirpath.joinpath("udhr"))
    udhr.download(force=force)
    udhr_data = [
        (snippet, meta["lang"])
        for text, meta in udhr.records()
        for snippet in text.split("\n")
        if meta["lang"] in valid_langs and
        itertoolz.count(char for char in snippet if char.isalnum()) >= min_len
    ]

    dslcc = textacy.lang_id_._datasets.DSLCCDataset(root_dirpath.joinpath("dslcc"))
    dslcc.download(force=force)
    dslcc_data = dslcc.load(valid_langs, min_len=min_len)

    wili = textacy.lang_id_._datasets.Wili2018Dataset(root_dirpath.joinpath("wili"))
    wili.download(force=force)
    wili_data = wili.load(iso_lang_map, min_len=min_len)

    tatoeba = textacy.lang_id_._datasets.TatoebaDataset(root_dirpath.joinpath("tatoeba"))
    tatoeba.download(force=force)
    tatoeba_data = tatoeba.load(iso_lang_map, min_len=min_len)

    ud = textacy.lang_id_._datasets.UDDataset(root_dirpath.joinpath("ud"))
    ud.download(force=force)
    ud_data = ud.load(valid_langs, min_len=min_len)

    # aggregate and sample datasets
    agg_data = (
        udhr_data
        + wili_data
        + get_random_sample(tatoeba_data, 200000, stratify=True, random_state=42)
        + get_random_sample(ud_data, 200000, stratify=True, random_state=42)
        # add additional examples for hard-to-distinguish language groups
        + get_random_sample(dslcc_data, 50000, stratify=True, random_state=42)
        # add some extra english examples, since there's apparently a fair amount
        # of english sprinkled throughout other languages, causing meh performance
        + get_random_sample(
            [item for item in tatoeba_data if item[1] == "en"],
            10000,
            stratify=False,
            random_state=42,
        )
    )
    agg_data = filter_data_by_lang_count(agg_data, min_obs)
    return agg_data
def load_and_agg_data ( root_dirpath : pathlib . Path , force : bool , min_len : int , min_obs : int ) -> List [ Tuple [ str , str ] ] : iso_lang_resource = textacy . lang_id_ . _datasets . IsoLangResource ( root_dirpath . joinpath ( "iso-639" ) ) iso_lang_resource . download ( force = force ) iso_lang_map = iso_lang_resource . load ( exclude = { "sh" } ) # TODO: why exclude sh? valid_langs = set ( iso_lang_map . values ( ) ) udhr = textacy . datasets . UDHR ( root_dirpath . joinpath ( "udhr" ) ) udhr . download ( force = force ) udhr_data = [ ( snippet , meta [ "lang" ] ) for text , meta in udhr . records ( ) for snippet in text . split ( "\n" ) if meta [ "lang" ] in valid_langs and itertoolz . count ( char for char in snippet if char . isalnum ( ) ) >= min_len ] dslcc = textacy . lang_id_ . _datasets . DSLCCDataset ( root_dirpath . joinpath ( "dslcc" ) ) dslcc . download ( force = force ) dslcc_data = dslcc . load ( valid_langs , min_len = min_len ) wili = textacy . lang_id_ . _datasets . Wili2018Dataset ( root_dirpath . joinpath ( "wili" ) ) wili . download ( force = force ) wili_data = wili . load ( iso_lang_map , min_len = min_len ) tatoeba = textacy . lang_id_ . _datasets . TatoebaDataset ( root_dirpath . joinpath ( "tatoeba" ) ) tatoeba . download ( force = force ) tatoeba_data = tatoeba . load ( iso_lang_map , min_len = min_len ) ud = textacy . lang_id_ . _datasets . UDDataset ( root_dirpath . joinpath ( "ud" ) ) ud . download ( force = force ) ud_data = ud . load ( valid_langs , min_len = min_len ) # aggregate and sample datasets agg_data = ( udhr_data + wili_data + get_random_sample ( tatoeba_data , 200000 , stratify = True , random_state = 42 ) + get_random_sample ( ud_data , 200000 , stratify = True , random_state = 42 ) # add additional examples for hard-to-distinguish language groups + get_random_sample ( dslcc_data , 50000 , stratify = True , random_state = 42 ) # add some extra english examples, since there's apparently a fair amount # of english sprinkled throughout other languages, causing meh performance + get_random_sample ( [ item for item in tatoeba_data if item [ 1 ] == "en" ] , 10000 , stratify = False , random_state = 42 , ) ) agg_data = filter_data_by_lang_count ( agg_data , min_obs ) return agg_data