(filename)
return extracted_items
This is process_jsonlines method for intro-only processed_wikipedia file.
    The item example:
    {"id": "45668011", "url": "https://en.wikipedia.org/wiki?curid=45668011", "title": "Flouch Roundabout",
     "text": ["Flouch Roundabout is a roundabout near Penistone, South Yorkshire, England, where the A628 meets the A616."],
     "charoffset": [[[0, 6],...]]
     "text_with_links" : ["Flouch Roundabout is a roundabout near <a href=\"Penistone\">Penistone</a>,
     <a href=\"South%20Yorkshire\">South Yorkshire</a>, England, where the <a href=\"A628%20road\">A628</a>
     meets the <a href=\"A616%20road\">A616</a>."],
        "charoffset_with_links": [[[0, 6], ... [213, 214]]]}
This is process_jsonlines method for intro-only processed_wikipedia file.
    The item example:
    {"id": "45668011", "url": "https://en.wikipedia.org/wiki?curid=45668011", "title": "Flouch Roundabout",
     "text": ["Flouch Roundabout is a roundabout near Penistone, South Yorkshire, England, where the A628 meets the A616."],
     "charoffset": [[[0, 6],...]]
     "text_with_links" : ["Flouch Roundabout is a roundabout near <a href=\"Penistone\">Penistone</a>,
     <a href=\"South%20Yorkshire\">South Yorkshire</a>, England, where the <a href=\"A628%20road\">A628</a>
     meets the <a href=\"A616%20road\">A616</a>."],
        "charoffset_with_links": [[[0, 6], ... [213, 214]]]}
This is process_jsonlines method for intro - only processed_wikipedia file . The item example : { id : 45668011 url : https : // en . wikipedia . org / wiki?curid = 45668011 title : Flouch Roundabout text : [ Flouch Roundabout is a roundabout near Penistone South Yorkshire England where the A628 meets the A616 . ] charoffset : [[[ 0 6 ] ... ]] text_with_links : [ Flouch Roundabout is a roundabout near <a href = \ Penistone \ > Penistone< / a > <a href = \ South%20Yorkshire \ > South Yorkshire< / a > England where the <a href = \ A628%20road \ > A628< / a > meets the <a href = \ A616%20road \ > A616< / a > . ] charoffset_with_links : [[[ 0 6 ] ... [ 213 214 ]]] }
def process_jsonlines_hotpotqa(filename):
    """
    This is process_jsonlines method for intro-only processed_wikipedia file.
    The item example:
    {"id": "45668011", "url": "https://en.wikipedia.org/wiki?curid=45668011", "title": "Flouch Roundabout",
     "text": ["Flouch Roundabout is a roundabout near Penistone, South Yorkshire, England, where the A628 meets the A616."],
     "charoffset": [[[0, 6],...]]
     "text_with_links" : ["Flouch Roundabout is a roundabout near <a href=\"Penistone\">Penistone</a>,
     <a href=\"South%20Yorkshire\">South Yorkshire</a>, England, where the <a href=\"A628%20road\">A628</a>
     meets the <a href=\"A616%20road\">A616</a>."],
        "charoffset_with_links": [[[0, 6], ... [213, 214]]]}
    """
    # item should be nested list
    extracted_items = []
    with jsonlines.open(filename) as reader:
        for obj in reader:
            wiki_id = obj["id"]
            title = obj["title"]
            title_id = make_wiki_id(title, 0)
            plain_text = "\t".join(obj["text"])
            text_with_links = "\t".join(obj["text_with_links"])

            hyper_linked_titles = []
            hyper_linked_titles = find_hyper_linked_titles(text_with_links)
            if len(hyper_linked_titles) > 0:
                hyper_linked_titles_text = "\t".join(hyper_linked_titles)
            else:
                hyper_linked_titles_text = ""
            extracted_items.append({"wiki_id": wiki_id, "title": title_id,
                                    "plain_text": plain_text,
                                    "hyper_linked_titles": hyper_linked_titles_text,
                                    "original_title": title})

    return extracted_items
def process_jsonlines_hotpotqa ( filename ) : # item should be nested list extracted_items = [ ] with jsonlines . open ( filename ) as reader : for obj in reader : wiki_id = obj [ "id" ] title = obj [ "title" ] title_id = make_wiki_id ( title , 0 ) plain_text = "\t" . join ( obj [ "text" ] ) text_with_links = "\t" . join ( obj [ "text_with_links" ] ) hyper_linked_titles = [ ] hyper_linked_titles = find_hyper_linked_titles ( text_with_links ) if len ( hyper_linked_titles ) > 0 : hyper_linked_titles_text = "\t" . join ( hyper_linked_titles ) else : hyper_linked_titles_text = "" extracted_items . append ( { "wiki_id" : wiki_id , "title" : title_id , "plain_text" : plain_text , "hyper_linked_titles" : hyper_linked_titles_text , "original_title" : title } ) return extracted_items