(self, corpus_path, line_truncate, side, shard_size,
                 assoc_iter=None)
Args:
            corpus_path: the corpus file path.
            line_truncate: the maximum length of a line to read.
                            0 for unlimited.
            side: "src" or "tgt".
            shard_size: the shard size, 0 means not sharding the file.
            assoc_iter: if not None, it is the associate iterator that
                        this iterator should align its step with.
Args:
            corpus_path: the corpus file path.
            line_truncate: the maximum length of a line to read.
                            0 for unlimited.
            side: "src" or "tgt".
            shard_size: the shard size, 0 means not sharding the file.
            assoc_iter: if not None, it is the associate iterator that
                        this iterator should align its step with.
Args : corpus_path : the corpus file path . line_truncate : the maximum length of a line to read . 0 for unlimited . side : src or tgt . shard_size : the shard size 0 means not sharding the file . assoc_iter : if not None it is the associate iterator that this iterator should align its step with .
def __init__(self, corpus_path, line_truncate, side, shard_size,
                 assoc_iter=None):
        """
        Args:
            corpus_path: the corpus file path.
            line_truncate: the maximum length of a line to read.
                            0 for unlimited.
            side: "src" or "tgt".
            shard_size: the shard size, 0 means not sharding the file.
            assoc_iter: if not None, it is the associate iterator that
                        this iterator should align its step with.
        """
        try:
            # The codecs module seems to have bugs with seek()/tell(),
            # so we use io.open().
            self.corpus = io.open(corpus_path, "r", encoding="utf-8")
        except IOError:
            sys.stderr.write("Failed to open corpus file: %s" % corpus_path)
            sys.exit(1)

        self.line_truncate = line_truncate
        self.side = side
        self.shard_size = shard_size
        self.assoc_iter = assoc_iter
        self.last_pos = 0
        self.line_index = -1
        self.eof = False
def __init__ ( self , corpus_path , line_truncate , side , shard_size , assoc_iter = None ) : try : # The codecs module seems to have bugs with seek()/tell(), # so we use io.open(). self . corpus = io . open ( corpus_path , "r" , encoding = "utf-8" ) except IOError : sys . stderr . write ( "Failed to open corpus file: %s" % corpus_path ) sys . exit ( 1 ) self . line_truncate = line_truncate self . side = side self . shard_size = shard_size self . assoc_iter = assoc_iter self . last_pos = 0 self . line_index = - 1 self . eof = False