(text_iter, text_path, truncate, side)
return (examples_iter, num_feats)
Args:
            text_iter(iterator): an iterator (or None) that we can loop over
                to read examples.
                It may be an openned file, a string list etc...
            text_path(str): path to file or None
            path (str): location of a src or tgt file.
            truncate (int): maximum sequence length (0 for unlimited).
            side (str): "src" or "tgt".

        Returns:
            (example_dict iterator, num_feats) tuple.
Args:
            text_iter(iterator): an iterator (or None) that we can loop over
                to read examples.
                It may be an openned file, a string list etc...
            text_path(str): path to file or None
            path (str): location of a src or tgt file.
            truncate (int): maximum sequence length (0 for unlimited).
            side (str): "src" or "tgt".
Args : text_iter ( iterator ) : an iterator ( or None ) that we can loop over to read examples . It may be an openned file a string list etc ... text_path ( str ) : path to file or None path ( str ) : location of a src or tgt file . truncate ( int ) : maximum sequence length ( 0 for unlimited ) . side ( str ) : src or tgt .
def make_text_examples_nfeats_tpl(text_iter, text_path, truncate, side):
        """
        Args:
            text_iter(iterator): an iterator (or None) that we can loop over
                to read examples.
                It may be an openned file, a string list etc...
            text_path(str): path to file or None
            path (str): location of a src or tgt file.
            truncate (int): maximum sequence length (0 for unlimited).
            side (str): "src" or "tgt".

        Returns:
            (example_dict iterator, num_feats) tuple.
        """
        assert side in ['src', 'tgt']

        if text_iter is None:
            if text_path is not None:
                text_iter = TextDataset.make_text_iterator_from_file(text_path)
            else:
                return (None, 0)

        # All examples have same number of features, so we peek first one
        # to get the num_feats.
        examples_nfeats_iter = \
            TextDataset.make_examples(text_iter, truncate, side)

        first_ex = next(examples_nfeats_iter)
        num_feats = first_ex[1]

        # Chain back the first element - we only want to peek it.
        examples_nfeats_iter = chain([first_ex], examples_nfeats_iter)
        examples_iter = (ex for ex, nfeats in examples_nfeats_iter)

        return (examples_iter, num_feats)
def make_text_examples_nfeats_tpl ( text_iter , text_path , truncate , side ) : assert side in [ 'src' , 'tgt' ] if text_iter is None : if text_path is not None : text_iter = TextDataset . make_text_iterator_from_file ( text_path ) else : return ( None , 0 ) # All examples have same number of features, so we peek first one # to get the num_feats. examples_nfeats_iter = TextDataset . make_examples ( text_iter , truncate , side ) first_ex = next ( examples_nfeats_iter ) num_feats = first_ex [ 1 ] # Chain back the first element - we only want to peek it. examples_nfeats_iter = chain ( [ first_ex ] , examples_nfeats_iter ) examples_iter = ( ex for ex , nfeats in examples_nfeats_iter ) return ( examples_iter , num_feats )