(model_bert, tokenizer, nlu_t, hds, sql_vocab, max_seq_length)
return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, i_sql_vocab, \
           l_n, l_hpu, l_hs, l_input, \
           nlu_tt, t_to_tt_idx, tt_to_t_idx
s2s version. Treat SQL-tokens as pseudo-headers
    sql_vocab = ("sql select", "sql where", "sql and", "sql equal", "sql greater than", "sql less than")

    e.g.)
    Q: What is the name of the player with score greater than 15?
    H: Name of the player, score
    Input: [CLS], what, is, ...,
    [SEP], name, of, the, player, [SEP], score,
    [SEP] sql, select, [SEP], sql, where, [SEP], sql, and, [SEP], ...

    Here, input is tokenized further by WordPiece (WP) tokenizer and fed into BERT.

    INPUT
    :param model_bert:
    :param tokenizer: WordPiece toknizer
    :param nlu: Question
    :param nlu_t: CoreNLP tokenized nlu.
    :param hds: Headers
    :param hs_t: None or 1st-level tokenized headers
    :param max_seq_length: max input token length

    OUTPUT
    tokens: BERT input tokens
    nlu_tt: WP-tokenized input natural language questions
    orig_to_tok_index: map the index of 1st-level-token to the index of 2nd-level-token
    tok_to_orig_index: inverse map.
s2s version. Treat SQL-tokens as pseudo-headers
    sql_vocab = ("sql select", "sql where", "sql and", "sql equal", "sql greater than", "sql less than")
s2s version . Treat SQL - tokens as pseudo - headers sql_vocab = ( sql select sql where sql and sql equal sql greater than sql less than )
def get_bert_output_s2s(model_bert, tokenizer, nlu_t, hds, sql_vocab, max_seq_length):
    """
    s2s version. Treat SQL-tokens as pseudo-headers
    sql_vocab = ("sql select", "sql where", "sql and", "sql equal", "sql greater than", "sql less than")

    e.g.)
    Q: What is the name of the player with score greater than 15?
    H: Name of the player, score
    Input: [CLS], what, is, ...,
    [SEP], name, of, the, player, [SEP], score,
    [SEP] sql, select, [SEP], sql, where, [SEP], sql, and, [SEP], ...

    Here, input is tokenized further by WordPiece (WP) tokenizer and fed into BERT.

    INPUT
    :param model_bert:
    :param tokenizer: WordPiece toknizer
    :param nlu: Question
    :param nlu_t: CoreNLP tokenized nlu.
    :param hds: Headers
    :param hs_t: None or 1st-level tokenized headers
    :param max_seq_length: max input token length

    OUTPUT
    tokens: BERT input tokens
    nlu_tt: WP-tokenized input natural language questions
    orig_to_tok_index: map the index of 1st-level-token to the index of 2nd-level-token
    tok_to_orig_index: inverse map.

    """


    l_n = []
    l_hs = []  # The length of columns for each batch
    l_input = []
    input_ids = []
    tokens = []
    segment_ids = []
    input_mask = []

    i_nlu = []  # index to retreive the position of contextual vector later.
    i_hds = []
    i_sql_vocab = []

    doc_tokens = []
    nlu_tt = []

    t_to_tt_idx = []
    tt_to_t_idx = []
    for b, nlu_t1 in enumerate(nlu_t):

        hds1 = hds[b]
        l_hs.append(len(hds1))


        # 1. 2nd tokenization using WordPiece
        tt_to_t_idx1 = []  # number indicates where sub-token belongs to in 1st-level-tokens (here, CoreNLP).
        t_to_tt_idx1 = []  # orig_to_tok_idx[i] = start index of i-th-1st-level-token in all_tokens.
        nlu_tt1 = []  # all_doc_tokens[ orig_to_tok_idx[i] ] returns first sub-token segement of i-th-1st-level-token
        for (i, token) in enumerate(nlu_t1):
            t_to_tt_idx1.append(
                len(nlu_tt1))  # all_doc_tokens[ indicate the start position of original 'white-space' tokens.
            sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                tt_to_t_idx1.append(i)
                nlu_tt1.append(sub_token)  # all_doc_tokens are further tokenized using WordPiece tokenizer
        nlu_tt.append(nlu_tt1)
        tt_to_t_idx.append(tt_to_t_idx1)
        t_to_tt_idx.append(t_to_tt_idx1)

        l_n.append(len(nlu_tt1))
        #         hds1_all_tok = tokenize_hds1(tokenizer, hds1)



        # [CLS] nlu [SEP] col1 [SEP] col2 [SEP] ...col-n [SEP]
        # 2. Generate BERT inputs & indices.
        # Combine hds1 and sql_vocab
        tokens1, segment_ids1, i_sql_vocab1, i_nlu1, i_hds1 = generate_inputs_s2s(tokenizer, nlu_tt1, hds1, sql_vocab)

        # i_hds1
        input_ids1 = tokenizer.convert_tokens_to_ids(tokens1)

        # Input masks
        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask1 = [1] * len(input_ids1)

        # 3. Zero-pad up to the sequence length.
        l_input.append( len(input_ids1) )
        while len(input_ids1) < max_seq_length:
            input_ids1.append(0)
            input_mask1.append(0)
            segment_ids1.append(0)

        assert len(input_ids1) == max_seq_length
        assert len(input_mask1) == max_seq_length
        assert len(segment_ids1) == max_seq_length

        input_ids.append(input_ids1)
        tokens.append(tokens1)
        segment_ids.append(segment_ids1)
        input_mask.append(input_mask1)

        i_nlu.append(i_nlu1)
        i_hds.append(i_hds1)
        i_sql_vocab.append(i_sql_vocab1)

    # Convert to tensor
    all_input_ids = torch.tensor(input_ids, dtype=torch.long).to(device)
    all_input_mask = torch.tensor(input_mask, dtype=torch.long).to(device)
    all_segment_ids = torch.tensor(segment_ids, dtype=torch.long).to(device)

    # 4. Generate BERT output.
    all_encoder_layer, pooled_output = model_bert(all_input_ids, all_segment_ids, all_input_mask)

    # 5. generate l_hpu from i_hds
    l_hpu = gen_l_hpu(i_hds)

    return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, i_sql_vocab, \
           l_n, l_hpu, l_hs, l_input, \
           nlu_tt, t_to_tt_idx, tt_to_t_idx
def get_bert_output_s2s ( model_bert , tokenizer , nlu_t , hds , sql_vocab , max_seq_length ) : l_n = [ ] l_hs = [ ] # The length of columns for each batch l_input = [ ] input_ids = [ ] tokens = [ ] segment_ids = [ ] input_mask = [ ] i_nlu = [ ] # index to retreive the position of contextual vector later. i_hds = [ ] i_sql_vocab = [ ] doc_tokens = [ ] nlu_tt = [ ] t_to_tt_idx = [ ] tt_to_t_idx = [ ] for b , nlu_t1 in enumerate ( nlu_t ) : hds1 = hds [ b ] l_hs . append ( len ( hds1 ) ) # 1. 2nd tokenization using WordPiece tt_to_t_idx1 = [ ] # number indicates where sub-token belongs to in 1st-level-tokens (here, CoreNLP). t_to_tt_idx1 = [ ] # orig_to_tok_idx[i] = start index of i-th-1st-level-token in all_tokens. nlu_tt1 = [ ] # all_doc_tokens[ orig_to_tok_idx[i] ] returns first sub-token segement of i-th-1st-level-token for ( i , token ) in enumerate ( nlu_t1 ) : t_to_tt_idx1 . append ( len ( nlu_tt1 ) ) # all_doc_tokens[ indicate the start position of original 'white-space' tokens. sub_tokens = tokenizer . tokenize ( token ) for sub_token in sub_tokens : tt_to_t_idx1 . append ( i ) nlu_tt1 . append ( sub_token ) # all_doc_tokens are further tokenized using WordPiece tokenizer nlu_tt . append ( nlu_tt1 ) tt_to_t_idx . append ( tt_to_t_idx1 ) t_to_tt_idx . append ( t_to_tt_idx1 ) l_n . append ( len ( nlu_tt1 ) ) #         hds1_all_tok = tokenize_hds1(tokenizer, hds1) # [CLS] nlu [SEP] col1 [SEP] col2 [SEP] ...col-n [SEP] # 2. Generate BERT inputs & indices. # Combine hds1 and sql_vocab tokens1 , segment_ids1 , i_sql_vocab1 , i_nlu1 , i_hds1 = generate_inputs_s2s ( tokenizer , nlu_tt1 , hds1 , sql_vocab ) # i_hds1 input_ids1 = tokenizer . convert_tokens_to_ids ( tokens1 ) # Input masks # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask1 = [ 1 ] * len ( input_ids1 ) # 3. Zero-pad up to the sequence length. l_input . append ( len ( input_ids1 ) ) while len ( input_ids1 ) < max_seq_length : input_ids1 . append ( 0 ) input_mask1 . append ( 0 ) segment_ids1 . append ( 0 ) assert len ( input_ids1 ) == max_seq_length assert len ( input_mask1 ) == max_seq_length assert len ( segment_ids1 ) == max_seq_length input_ids . append ( input_ids1 ) tokens . append ( tokens1 ) segment_ids . append ( segment_ids1 ) input_mask . append ( input_mask1 ) i_nlu . append ( i_nlu1 ) i_hds . append ( i_hds1 ) i_sql_vocab . append ( i_sql_vocab1 ) # Convert to tensor all_input_ids = torch . tensor ( input_ids , dtype = torch . long ) . to ( device ) all_input_mask = torch . tensor ( input_mask , dtype = torch . long ) . to ( device ) all_segment_ids = torch . tensor ( segment_ids , dtype = torch . long ) . to ( device ) # 4. Generate BERT output. all_encoder_layer , pooled_output = model_bert ( all_input_ids , all_segment_ids , all_input_mask ) # 5. generate l_hpu from i_hds l_hpu = gen_l_hpu ( i_hds ) return all_encoder_layer , pooled_output , tokens , i_nlu , i_hds , i_sql_vocab , l_n , l_hpu , l_hs , l_input , nlu_tt , t_to_tt_idx , tt_to_t_idx