(self)
Iterator of (example_dict, nfeats).
        On each call, it iterates over as many (example_dict, nfeats) tuples
        until this shard's size equals to or approximates `self.shard_size`.
Iterator of (example_dict, nfeats).
        On each call, it iterates over as many (example_dict, nfeats) tuples
        until this shard's size equals to or approximates `self.shard_size`.
Iterator of ( example_dict nfeats ) . On each call it iterates over as many ( example_dict nfeats ) tuples until this shard s size equals to or approximates self . shard_size .
def __iter__(self):
        """
        Iterator of (example_dict, nfeats).
        On each call, it iterates over as many (example_dict, nfeats) tuples
        until this shard's size equals to or approximates `self.shard_size`.
        """
        iteration_index = -1
        if self.assoc_iter is not None:
            # We have associate iterator, just yields tuples
            # util we run parallel with it.
            while self.line_index < self.assoc_iter.line_index:
                line = self.corpus.readline()
                if line == '':
                    raise AssertionError(
                        "Two corpuses must have same number of lines!")

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)

            if self.assoc_iter.eof:
                self.eof = True
                self.corpus.close()
        else:
            # Yield tuples util this shard's size reaches the threshold.
            self.corpus.seek(self.last_pos)
            while True:
                if self.shard_size != 0 and self.line_index % 64 == 0:
                    # This part of check is time consuming on Py2 (but
                    # it is quite fast on Py3, weird!). So we don't bother
                    # to check for very line. Instead we chekc every 64
                    # lines. Thus we are not dividing exactly per
                    # `shard_size`, but it is not too much difference.
                    cur_pos = self.corpus.tell()
                    if cur_pos >= self.last_pos + self.shard_size:
                        self.last_pos = cur_pos
                        return

                line = self.corpus.readline()
                if line == '':
                    self.eof = True
                    self.corpus.close()
                    return

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)
def __iter__ ( self ) : iteration_index = - 1 if self . assoc_iter is not None : # We have associate iterator, just yields tuples # util we run parallel with it. while self . line_index < self . assoc_iter . line_index : line = self . corpus . readline ( ) if line == '' : raise AssertionError ( "Two corpuses must have same number of lines!" ) self . line_index += 1 iteration_index += 1 yield self . _example_dict_iter ( line , iteration_index ) if self . assoc_iter . eof : self . eof = True self . corpus . close ( ) else : # Yield tuples util this shard's size reaches the threshold. self . corpus . seek ( self . last_pos ) while True : if self . shard_size != 0 and self . line_index % 64 == 0 : # This part of check is time consuming on Py2 (but # it is quite fast on Py3, weird!). So we don't bother # to check for very line. Instead we chekc every 64 # lines. Thus we are not dividing exactly per # `shard_size`, but it is not too much difference. cur_pos = self . corpus . tell ( ) if cur_pos >= self . last_pos + self . shard_size : self . last_pos = cur_pos return line = self . corpus . readline ( ) if line == '' : self . eof = True self . corpus . close ( ) return self . line_index += 1 iteration_index += 1 yield self . _example_dict_iter ( line , iteration_index )