(self)
return {w for q in self._dataset_tagged for w, _, _ in q}
Generate a list of tokens that appear in question in the complete dataset.

        :return: set tokens
Generate a list of tokens that appear in question in the complete dataset.
Generate a list of tokens that appear in question in the complete dataset .
def get_question_tokens_set(self):
        """
        Generate a list of tokens that appear in question in the complete dataset.

        :return: set tokens
        """
        if self._p.get("normalize.tokens", False):
            return {re.sub(r"\d+", "<n>", w.lower()) for q in self._dataset_tagged for w, _, _ in q}
        return {w for q in self._dataset_tagged for w, _, _ in q}
def get_question_tokens_set ( self ) : if self . _p . get ( "normalize.tokens" , False ) : return { re . sub ( r"\d+" , "<n>" , w . lower ( ) ) for q in self . _dataset_tagged for w , _ , _ in q } return { w for q in self . _dataset_tagged for w , _ , _ in q }