(parser)
Pre-procesing options
Pre-procesing options
Pre - procesing options
def preprocess_opts(parser):
    """ Pre-procesing options """
    # Data options
    group = parser.add_argument_group('Data')
    group.add_argument('-data_type', default="text",
                       help="""Type of the source input.
                       Options are [text|img].""")

    group.add_argument('-train_src', required=True,
                       help="Path to the training source data")
    group.add_argument('-train_tgt', required=True,
                       help="Path to the training target data")
    group.add_argument('-valid_src', required=True,
                       help="Path to the validation source data")
    group.add_argument('-valid_tgt', required=True,
                       help="Path to the validation target data")

    group.add_argument('-src_dir', default="",
                       help="Source directory for image or audio files.")

    group.add_argument('-save_data', required=True,
                       help="Output file for the prepared data")

    group.add_argument('-max_shard_size', type=int, default=0,
                       help="""For text corpus of large volume, it will
                       be divided into shards of this size to preprocess.
                       If 0, the data will be handled as a whole. The unit
                       is in bytes. Optimal value should be multiples of
                       64 bytes. A commonly used sharding value is 131072000.
                       It is recommended to ensure the corpus is shuffled
                       before sharding.""")

    group.add_argument('-shard_size', type=int, default=0,
                       help="""Divide src_corpus and tgt_corpus into
                       smaller multiple src_copus and tgt corpus files, then
                       build shards, each shard will have
                       opt.shard_size samples except last shard.
                       shard_size=0 means no segmentation
                       shard_size>0 means segment dataset into multiple shards,
                       each shard has shard_size samples""")

    # Dictionary options, for text corpus

    group = parser.add_argument_group('Vocab')
    group.add_argument('-src_vocab', default="",
                       help="""Path to an existing source vocabulary. Format:
                       one word per line.""")
    group.add_argument('-tgt_vocab', default="",
                       help="""Path to an existing target vocabulary. Format:
                       one word per line.""")
    group.add_argument('-features_vocabs_prefix', type=str, default='',
                       help="Path prefix to existing features vocabularies")
    group.add_argument('-src_vocab_size', type=int, default=50000,
                       help="Size of the source vocabulary")
    group.add_argument('-tgt_vocab_size', type=int, default=50000,
                       help="Size of the target vocabulary")

    group.add_argument('-src_words_min_frequency', type=int, default=0)
    group.add_argument('-tgt_words_min_frequency', type=int, default=0)

    group.add_argument('-dynamic_dict', action='store_true',
                       help="Create dynamic dictionaries")
    group.add_argument('-share_vocab', action='store_true',
                       help="Share source and target vocabulary")

    # Truncation options, for text corpus
    group = parser.add_argument_group('Pruning')
    group.add_argument('-src_seq_length', type=int, default=50,
                       help="Maximum source sequence length")
    group.add_argument('-src_seq_length_trunc', type=int, default=0,
                       help="Truncate source sequence length.")
    group.add_argument('-tgt_seq_length', type=int, default=50,
                       help="Maximum target sequence length to keep.")
    group.add_argument('-tgt_seq_length_trunc', type=int, default=0,
                       help="Truncate target sequence length.")
    group.add_argument('-lower', action='store_true', help='lowercase data')

    # Data processing options
    group = parser.add_argument_group('Random')
    group.add_argument('-shuffle', type=int, default=1,
                       help="Shuffle data")
    group.add_argument('-seed', type=int, default=3435,
                       help="Random seed")

    group = parser.add_argument_group('Logging')
    group.add_argument('-report_every', type=int, default=100000,
                       help="Report status every this many sentences")
    group.add_argument('-log_file', type=str, default="",
                       help="Output logs to a file under this path.")

    # Options most relevant to speech
    group = parser.add_argument_group('Speech')
    group.add_argument('-sample_rate', type=int, default=16000,
                       help="Sample rate.")
    group.add_argument('-window_size', type=float, default=.02,
                       help="Window size for spectrogram in seconds.")
    group.add_argument('-window_stride', type=float, default=.01,
                       help="Window stride for spectrogram in seconds.")
    group.add_argument('-window', default='hamming',
                       help="Window type for spectrogram generation.")

    # Option most relevant to image input
    group.add_argument('-image_channel_size', type=int, default=3,
                       choices=[3, 1],
                       help="""Using grayscale image can training
                       model faster and smaller""")
def preprocess_opts ( parser ) : # Data options group = parser . add_argument_group ( 'Data' ) group . add_argument ( '-data_type' , default = "text" , help = """Type of the source input.
                       Options are [text|img].""" ) group . add_argument ( '-train_src' , required = True , help = "Path to the training source data" ) group . add_argument ( '-train_tgt' , required = True , help = "Path to the training target data" ) group . add_argument ( '-valid_src' , required = True , help = "Path to the validation source data" ) group . add_argument ( '-valid_tgt' , required = True , help = "Path to the validation target data" ) group . add_argument ( '-src_dir' , default = "" , help = "Source directory for image or audio files." ) group . add_argument ( '-save_data' , required = True , help = "Output file for the prepared data" ) group . add_argument ( '-max_shard_size' , type = int , default = 0 , help = """For text corpus of large volume, it will
                       be divided into shards of this size to preprocess.
                       If 0, the data will be handled as a whole. The unit
                       is in bytes. Optimal value should be multiples of
                       64 bytes. A commonly used sharding value is 131072000.
                       It is recommended to ensure the corpus is shuffled
                       before sharding.""" ) group . add_argument ( '-shard_size' , type = int , default = 0 , help = """Divide src_corpus and tgt_corpus into
                       smaller multiple src_copus and tgt corpus files, then
                       build shards, each shard will have
                       opt.shard_size samples except last shard.
                       shard_size=0 means no segmentation
                       shard_size>0 means segment dataset into multiple shards,
                       each shard has shard_size samples""" ) # Dictionary options, for text corpus group = parser . add_argument_group ( 'Vocab' ) group . add_argument ( '-src_vocab' , default = "" , help = """Path to an existing source vocabulary. Format:
                       one word per line.""" ) group . add_argument ( '-tgt_vocab' , default = "" , help = """Path to an existing target vocabulary. Format:
                       one word per line.""" ) group . add_argument ( '-features_vocabs_prefix' , type = str , default = '' , help = "Path prefix to existing features vocabularies" ) group . add_argument ( '-src_vocab_size' , type = int , default = 50000 , help = "Size of the source vocabulary" ) group . add_argument ( '-tgt_vocab_size' , type = int , default = 50000 , help = "Size of the target vocabulary" ) group . add_argument ( '-src_words_min_frequency' , type = int , default = 0 ) group . add_argument ( '-tgt_words_min_frequency' , type = int , default = 0 ) group . add_argument ( '-dynamic_dict' , action = 'store_true' , help = "Create dynamic dictionaries" ) group . add_argument ( '-share_vocab' , action = 'store_true' , help = "Share source and target vocabulary" ) # Truncation options, for text corpus group = parser . add_argument_group ( 'Pruning' ) group . add_argument ( '-src_seq_length' , type = int , default = 50 , help = "Maximum source sequence length" ) group . add_argument ( '-src_seq_length_trunc' , type = int , default = 0 , help = "Truncate source sequence length." ) group . add_argument ( '-tgt_seq_length' , type = int , default = 50 , help = "Maximum target sequence length to keep." ) group . add_argument ( '-tgt_seq_length_trunc' , type = int , default = 0 , help = "Truncate target sequence length." ) group . add_argument ( '-lower' , action = 'store_true' , help = 'lowercase data' ) # Data processing options group = parser . add_argument_group ( 'Random' ) group . add_argument ( '-shuffle' , type = int , default = 1 , help = "Shuffle data" ) group . add_argument ( '-seed' , type = int , default = 3435 , help = "Random seed" ) group = parser . add_argument_group ( 'Logging' ) group . add_argument ( '-report_every' , type = int , default = 100000 , help = "Report status every this many sentences" ) group . add_argument ( '-log_file' , type = str , default = "" , help = "Output logs to a file under this path." ) # Options most relevant to speech group = parser . add_argument_group ( 'Speech' ) group . add_argument ( '-sample_rate' , type = int , default = 16000 , help = "Sample rate." ) group . add_argument ( '-window_size' , type = float , default = .02 , help = "Window size for spectrogram in seconds." ) group . add_argument ( '-window_stride' , type = float , default = .01 , help = "Window stride for spectrogram in seconds." ) group . add_argument ( '-window' , default = 'hamming' , help = "Window type for spectrogram generation." ) # Option most relevant to image input group . add_argument ( '-image_channel_size' , type = int , default = 3 , choices = [ 3 , 1 ] , help = """Using grayscale image can training
                       model faster and smaller""" )