(tokenizer, nlu1_tt, hds1, sql_vocab1)
return tokens, segment_ids, i_sql_vocab, i_nlu, i_hds
[CLS] sql_vocab [SEP] question [SEP] headers
    To make sql_vocab in a fixed position.
[CLS] sql_vocab [SEP] question [SEP] headers
    To make sql_vocab in a fixed position.
[ CLS ] sql_vocab [ SEP ] question [ SEP ] headers To make sql_vocab in a fixed position .
def generate_inputs_s2s(tokenizer, nlu1_tt, hds1, sql_vocab1):
    """
    [CLS] sql_vocab [SEP] question [SEP] headers
    To make sql_vocab in a fixed position.
    """

    tokens = []
    segment_ids = []

    tokens.append("[CLS]")


    # sql_vocab
    i_sql_vocab = []
    # for doc
    for i, sql_vocab11 in enumerate(sql_vocab1):
        i_st_sql = len(tokens)
        sub_tok = tokenizer.tokenize(sql_vocab11)
        tokens += sub_tok
        i_ed_sql = len(tokens)
        i_sql_vocab.append((i_st_sql, i_ed_sql))
        segment_ids += [1] * len(sub_tok)
        if i < len(sql_vocab1) - 1:
            tokens.append("[SEP]")
            segment_ids.append(0)
        elif i == len(sql_vocab1) - 1:
            tokens.append("[SEP]")
            segment_ids.append(1)
        else:
            raise EnvironmentError


    # question
    i_st_nlu = len(tokens)  # to use it later

    segment_ids.append(0)
    for token in nlu1_tt:
        tokens.append(token)
        segment_ids.append(0)
    i_ed_nlu = len(tokens)
    tokens.append("[SEP]")
    segment_ids.append(0)
    i_nlu = (i_st_nlu, i_ed_nlu)


    # headers
    i_hds = []
    # for doc
    for i, hds11 in enumerate(hds1):
        i_st_hd = len(tokens)
        sub_tok = tokenizer.tokenize(hds11)
        tokens += sub_tok
        i_ed_hd = len(tokens)
        i_hds.append((i_st_hd, i_ed_hd))
        segment_ids += [1] * len(sub_tok)
        if i < len(hds1)-1:
            tokens.append("[SEP]")
            segment_ids.append(0)
        elif i == len(hds1)-1:
            tokens.append("[SEP]")
            segment_ids.append(1)
        else:
            raise EnvironmentError


    return tokens, segment_ids, i_sql_vocab, i_nlu, i_hds
def generate_inputs_s2s ( tokenizer , nlu1_tt , hds1 , sql_vocab1 ) : tokens = [ ] segment_ids = [ ] tokens . append ( "[CLS]" ) # sql_vocab i_sql_vocab = [ ] # for doc for i , sql_vocab11 in enumerate ( sql_vocab1 ) : i_st_sql = len ( tokens ) sub_tok = tokenizer . tokenize ( sql_vocab11 ) tokens += sub_tok i_ed_sql = len ( tokens ) i_sql_vocab . append ( ( i_st_sql , i_ed_sql ) ) segment_ids += [ 1 ] * len ( sub_tok ) if i < len ( sql_vocab1 ) - 1 : tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) elif i == len ( sql_vocab1 ) - 1 : tokens . append ( "[SEP]" ) segment_ids . append ( 1 ) else : raise EnvironmentError # question i_st_nlu = len ( tokens ) # to use it later segment_ids . append ( 0 ) for token in nlu1_tt : tokens . append ( token ) segment_ids . append ( 0 ) i_ed_nlu = len ( tokens ) tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) i_nlu = ( i_st_nlu , i_ed_nlu ) # headers i_hds = [ ] # for doc for i , hds11 in enumerate ( hds1 ) : i_st_hd = len ( tokens ) sub_tok = tokenizer . tokenize ( hds11 ) tokens += sub_tok i_ed_hd = len ( tokens ) i_hds . append ( ( i_st_hd , i_ed_hd ) ) segment_ids += [ 1 ] * len ( sub_tok ) if i < len ( hds1 ) - 1 : tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) elif i == len ( hds1 ) - 1 : tokens . append ( "[SEP]" ) segment_ids . append ( 1 ) else : raise EnvironmentError return tokens , segment_ids , i_sql_vocab , i_nlu , i_hds