(
        self, tokenized_docs: Iterable[Iterable[str]], fixed_vocab: bool
    )
return doc_term_matrix, vocabulary
Count terms found in ``tokenized_docs`` and, if ``fixed_vocab`` is False,
        build up a vocabulary based on those terms.

        Args:
            tokenized_docs
            fixed_vocab

        Returns:
            Document-term matrix and vocabulary used to make it.
Count terms found in ``tokenized_docs`` and, if ``fixed_vocab`` is False,
        build up a vocabulary based on those terms.
Count terms found in tokenized_docs and if fixed_vocab is False build up a vocabulary based on those terms .
def _count_terms(
        self, tokenized_docs: Iterable[Iterable[str]], fixed_vocab: bool
    ) -> Tuple[sp.csr_matrix, Dict[str, int]]:
        """
        Count terms found in ``tokenized_docs`` and, if ``fixed_vocab`` is False,
        build up a vocabulary based on those terms.

        Args:
            tokenized_docs
            fixed_vocab

        Returns:
            Document-term matrix and vocabulary used to make it.
        """
        if fixed_vocab is False:
            # add a new value when a new term is seen
            vocabulary = collections.defaultdict()
            vocabulary.default_factory = vocabulary.__len__
        else:
            vocabulary = self.vocabulary_terms

        indices = array(str("i"))
        indptr = array(str("i"), [0])
        for terms in tokenized_docs:
            for term in terms:
                try:
                    indices.append(vocabulary[term])
                except KeyError:
                    # ignore out-of-vocabulary terms when _fixed_terms=True
                    continue
            indptr.append(len(indices))

        if fixed_vocab is False:
            # we no longer want defaultdict behaviour
            vocabulary = dict(vocabulary)

        indices = np.frombuffer(indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        data = np.ones(len(indices))

        # build the matrix, then consolidate duplicate entries
        # by adding them together, in-place
        doc_term_matrix = sp.csr_matrix(
            (data, indices, indptr),
            shape=(len(indptr) - 1, len(vocabulary)),
            dtype=np.int32,
        )
        doc_term_matrix.sum_duplicates()

        # pretty sure this is a good thing to do... o_O
        doc_term_matrix.sort_indices()

        return doc_term_matrix, vocabulary
def _count_terms ( self , tokenized_docs : Iterable [ Iterable [ str ] ] , fixed_vocab : bool ) -> Tuple [ sp . csr_matrix , Dict [ str , int ] ] : if fixed_vocab is False : # add a new value when a new term is seen vocabulary = collections . defaultdict ( ) vocabulary . default_factory = vocabulary . __len__ else : vocabulary = self . vocabulary_terms indices = array ( str ( "i" ) ) indptr = array ( str ( "i" ) , [ 0 ] ) for terms in tokenized_docs : for term in terms : try : indices . append ( vocabulary [ term ] ) except KeyError : # ignore out-of-vocabulary terms when _fixed_terms=True continue indptr . append ( len ( indices ) ) if fixed_vocab is False : # we no longer want defaultdict behaviour vocabulary = dict ( vocabulary ) indices = np . frombuffer ( indices , dtype = np . intc ) indptr = np . frombuffer ( indptr , dtype = np . intc ) data = np . ones ( len ( indices ) ) # build the matrix, then consolidate duplicate entries # by adding them together, in-place doc_term_matrix = sp . csr_matrix ( ( data , indices , indptr ) , shape = ( len ( indptr ) - 1 , len ( vocabulary ) ) , dtype = np . int32 , ) doc_term_matrix . sum_duplicates ( ) # pretty sure this is a good thing to do... o_O doc_term_matrix . sort_indices ( ) return doc_term_matrix , vocabulary