(dirpath, langs, min_len=25)
return sorted(set(data), key=operator.itemgetter(1))
Args:
        dirpath (str)
        langs (Set[str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]
Args:
        dirpath (str)
        langs (Set[str])
        min_len (int): minimum text length in *chars*
Args : dirpath ( str ) langs ( Set [ str ] ) min_len ( int ) : minimum text length in * chars *
def load_dslcc_data(dirpath, langs, min_len=25):
    """
    Args:
        dirpath (str)
        langs (Set[str])
        min_len (int): minimum text length in *chars*

    Returns:
        List[Tuple[str, str]]
    """
    data = []
    fstubs = [
        "dslcc3/train/task1-train.txt",
        "dslcc3/train/task1-dev.txt",
        "dslcc4/DSL-TRAIN.txt",
        "dslcc4/DSL-DEV.txt",
    ]
    for fstub in fstubs:
        filepath = dirpath.joinpath(fstub)
        lines = textacy.io.read_text(filepath, mode="rt", encoding="utf-8", lines=True)
        for line in lines:
            if not line.strip():
                continue
            try:
                text, lang = line.split("\t")
                if len(text) >= min_len and lang[:2] in langs:
                    data.append((text, lang[:2]))
            except Exception:
                logging.warning("bad line in data")
                pass
    return sorted(set(data), key=operator.itemgetter(1))
def load_dslcc_data ( dirpath , langs , min_len = 25 ) : data = [ ] fstubs = [ "dslcc3/train/task1-train.txt" , "dslcc3/train/task1-dev.txt" , "dslcc4/DSL-TRAIN.txt" , "dslcc4/DSL-DEV.txt" , ] for fstub in fstubs : filepath = dirpath . joinpath ( fstub ) lines = textacy . io . read_text ( filepath , mode = "rt" , encoding = "utf-8" , lines = True ) for line in lines : if not line . strip ( ) : continue try : text , lang = line . split ( "\t" ) if len ( text ) >= min_len and lang [ : 2 ] in langs : data . append ( ( text , lang [ : 2 ] ) ) except Exception : logging . warning ( "bad line in data" ) pass return sorted ( set ( data ) , key = operator . itemgetter ( 1 ) )