(dirpath, creds_fpath, force=False)
Download two collections of (lang, tweet_id) pairs from Twitter --
    a "uniformly sampled" collection of ~120k tweets over all languages and
    a "recall oriented" collection of ~1.5k tweets per language --
    then fetch available tweets' data from the Twitter API.

    Args:
        dirpath (str or :class:`pathlib.Path`)
        creds_fpath (str or :class:`pathlib.Path`)
        force (bool)

    References:
        https://blog.twitter.com/engineering/en_us/a/2015/evaluating-language-identification-performance.html

    TODO: Ideally, use a tweet search endpoint and filter by language,
    then just iterate over all ISO-639-1 language codes.
Download two collections of (lang, tweet_id) pairs from Twitter --
    a "uniformly sampled" collection of ~120k tweets over all languages and
    a "recall oriented" collection of ~1.5k tweets per language --
    then fetch available tweets' data from the Twitter API.
Download two collections of ( lang tweet_id ) pairs from Twitter -- a uniformly sampled collection of ~120k tweets over all languages and a recall oriented collection of ~1 . 5k tweets per language -- then fetch available tweets data from the Twitter API .
def download_twitter_data(dirpath, creds_fpath, force=False):
    """
    Download two collections of (lang, tweet_id) pairs from Twitter --
    a "uniformly sampled" collection of ~120k tweets over all languages and
    a "recall oriented" collection of ~1.5k tweets per language --
    then fetch available tweets' data from the Twitter API.

    Args:
        dirpath (str or :class:`pathlib.Path`)
        creds_fpath (str or :class:`pathlib.Path`)
        force (bool)

    References:
        https://blog.twitter.com/engineering/en_us/a/2015/evaluating-language-identification-performance.html

    TODO: Ideally, use a tweet search endpoint and filter by language,
    then just iterate over all ISO-639-1 language codes.
    """
    dirpath = textacy.utils.to_path(dirpath).resolve()
    url_fnames = [
        (
            "https://raw.githubusercontent.com/mitjat/langid_eval/master/uniformly_sampled.tsv",
            "uniformly_sampled.tsv",
        ),
        (
            "https://raw.githubusercontent.com/mitjat/langid_eval/master/recall_oriented.tsv",
            "recall_oriented.tsv",
        ),
    ]
    # download tweet ids first
    for url, fname in url_fnames:
        textacy.io.download_file(url, filename=fname, dirpath=dirpath, force=force)
    # download full tweets data next
    tweets_fpath = dirpath.joinpath("tweets.jsonl")
    if tweets_fpath.is_file() and force is False:
        logging.info("tweets data already downloaded to %s", tweets_fpath)
        return

    # load twitter ids data from disk
    tweet_lang_ids = []
    for fname in ["uniformly_sampled.tsv", "recall_oriented.tsv"]:
        tweet_lang_ids.extend(
            textacy.io.read_csv(
                dirpath.joinpath(fname),
                delimiter="\t",
                fieldnames=["lang", "status_id"],
                quoting=1,
            )
        )
    logging.info("loaded %s tweet ids from disk", len(tweet_lang_ids))
    # parse status ids
    status_ids = set()
    for row in tweet_lang_ids:
        try:
            status_ids.add(int(row["status_id"]))
        # there are a small handful of bad status ids, shrug
        except ValueError:
            pass
    logging.info("... of which %s had valid, unique ids", len(status_ids))
    status_ids = list(status_ids)
    # instantiate twitter api client
    with textacy.utils.to_path(creds_fpath).resolve().open(mode="rt") as f:
        creds = yaml.safe_load(f.read())
    api = twitter.Api(sleep_on_rate_limit=True, **creds)
    # get tweets data in chunks
    chunk_size = 100
    pbar = tqdm.tqdm(total=len(status_ids), unit="tweets")
    tweets = []
    try:
        for chunk_ids in itertoolz.partition_all(chunk_size, status_ids):
            chunk_tweets = api.GetStatuses(
                chunk_ids, trim_user=True, include_entities=True, map=False
            )
            tweets.extend(chunk_tweets)
            pbar.update(len(chunk_ids))
    except Exception:
        logging.exception("encountered an error while downloading tweets")
    finally:
        pbar.close()
        tweets = [tweet.AsDict() for tweet in tweets]
        logging.info("downloaded data for %s tweets", len(tweets))
        textacy.io.write_json(tweets, tweets_fpath, mode="wt", lines=True)
def download_twitter_data ( dirpath , creds_fpath , force = False ) : dirpath = textacy . utils . to_path ( dirpath ) . resolve ( ) url_fnames = [ ( "https://raw.githubusercontent.com/mitjat/langid_eval/master/uniformly_sampled.tsv" , "uniformly_sampled.tsv" , ) , ( "https://raw.githubusercontent.com/mitjat/langid_eval/master/recall_oriented.tsv" , "recall_oriented.tsv" , ) , ] # download tweet ids first for url , fname in url_fnames : textacy . io . download_file ( url , filename = fname , dirpath = dirpath , force = force ) # download full tweets data next tweets_fpath = dirpath . joinpath ( "tweets.jsonl" ) if tweets_fpath . is_file ( ) and force is False : logging . info ( "tweets data already downloaded to %s" , tweets_fpath ) return # load twitter ids data from disk tweet_lang_ids = [ ] for fname in [ "uniformly_sampled.tsv" , "recall_oriented.tsv" ] : tweet_lang_ids . extend ( textacy . io . read_csv ( dirpath . joinpath ( fname ) , delimiter = "\t" , fieldnames = [ "lang" , "status_id" ] , quoting = 1 , ) ) logging . info ( "loaded %s tweet ids from disk" , len ( tweet_lang_ids ) ) # parse status ids status_ids = set ( ) for row in tweet_lang_ids : try : status_ids . add ( int ( row [ "status_id" ] ) ) # there are a small handful of bad status ids, shrug except ValueError : pass logging . info ( "... of which %s had valid, unique ids" , len ( status_ids ) ) status_ids = list ( status_ids ) # instantiate twitter api client with textacy . utils . to_path ( creds_fpath ) . resolve ( ) . open ( mode = "rt" ) as f : creds = yaml . safe_load ( f . read ( ) ) api = twitter . Api ( sleep_on_rate_limit = True , * * creds ) # get tweets data in chunks chunk_size = 100 pbar = tqdm . tqdm ( total = len ( status_ids ) , unit = "tweets" ) tweets = [ ] try : for chunk_ids in itertoolz . partition_all ( chunk_size , status_ids ) : chunk_tweets = api . GetStatuses ( chunk_ids , trim_user = True , include_entities = True , map = False ) tweets . extend ( chunk_tweets ) pbar . update ( len ( chunk_ids ) ) except Exception : logging . exception ( "encountered an error while downloading tweets" ) finally : pbar . close ( ) tweets = [ tweet . AsDict ( ) for tweet in tweets ] logging . info ( "downloaded data for %s tweets" , len ( tweets ) ) textacy . io . write_json ( tweets , tweets_fpath , mode = "wt" , lines = True )