(ngram, hash_size, multi_para, doc_id)
return row, col, data
Fetch the text of a document and compute hashed ngrams counts.
Fetch the text of a document and compute hashed ngrams counts.
Fetch the text of a document and compute hashed ngrams counts .
def count(ngram, hash_size, multi_para, doc_id):
    """Fetch the text of a document and compute hashed ngrams counts."""
    global DOC2IDX
    # FIXME: remove hard coding.
    row, col, data = [], [], []
    # Tokenize
    
    if multi_para is True:
        # 1. if multi_para is true, the doc contains multiple paragraphs separated by \n\n and with links.
        tokens = tokenize(fetch_text_multi_para(doc_id))
    else:
        # 2. if not, only intro docs are retrieved and the sentences are separated by \t.
        # remove sentence separations ("\t") (only for HotpotQA).
        tokens = tokenize(fetch_text(doc_id).replace("\t", ""))

    # Get ngrams from tokens, with stopword/punctuation filtering.
    ngrams = tokens.ngrams(
        n=ngram, uncased=True, filter_fn=filter_ngram
    )

    # Hash ngrams and count occurences
    counts = Counter([hash(gram, hash_size)
                      for gram in ngrams])

    # Return in sparse matrix data format.
    row.extend(counts.keys())
    col.extend([DOC2IDX[doc_id]] * len(counts))
    data.extend(counts.values())
    return row, col, data
def count ( ngram , hash_size , multi_para , doc_id ) : global DOC2IDX # FIXME: remove hard coding. row , col , data = [ ] , [ ] , [ ] # Tokenize if multi_para is True : # 1. if multi_para is true, the doc contains multiple paragraphs separated by \n\n and with links. tokens = tokenize ( fetch_text_multi_para ( doc_id ) ) else : # 2. if not, only intro docs are retrieved and the sentences are separated by \t. # remove sentence separations ("\t") (only for HotpotQA). tokens = tokenize ( fetch_text ( doc_id ) . replace ( "\t" , "" ) ) # Get ngrams from tokens, with stopword/punctuation filtering. ngrams = tokens . ngrams ( n = ngram , uncased = True , filter_fn = filter_ngram ) # Hash ngrams and count occurences counts = Counter ( [ hash ( gram , hash_size ) for gram in ngrams ] ) # Return in sparse matrix data format. row . extend ( counts . keys ( ) ) col . extend ( [ DOC2IDX [ doc_id ] ] * len ( counts ) ) data . extend ( counts . values ( ) ) return row , col , data