(path_wikisql, mode='train', toy_model=False, toy_size=10, no_hs_tok=False, aug=False)
return data, table
Load training sets
Load training sets
Load training sets
def load_wikisql_data(path_wikisql, mode='train', toy_model=False, toy_size=10, no_hs_tok=False, aug=False):
    """ Load training sets
    """
    if aug:
        mode = f"aug.{mode}"
        print('Augmented data is loaded!')

    path_sql = os.path.join(path_wikisql, mode+'_tok.jsonl')
    if no_hs_tok:
        path_table = os.path.join(path_wikisql, mode + '.tables.jsonl')
    else:
        path_table = os.path.join(path_wikisql, mode+'_tok.tables.jsonl')

    data = []
    table = {}
    with open(path_sql) as f:
        for idx, line in enumerate(f):
            if toy_model and idx >= toy_size:
                break

            t1 = json.loads(line.strip())
            data.append(t1)

    with open(path_table) as f:
        for idx, line in enumerate(f):
            if toy_model and idx > toy_size:
                break

            t1 = json.loads(line.strip())
            table[t1['id']] = t1

    return data, table
def load_wikisql_data ( path_wikisql , mode = 'train' , toy_model = False , toy_size = 10 , no_hs_tok = False , aug = False ) : if aug : mode = f"aug.{mode}" print ( 'Augmented data is loaded!' ) path_sql = os . path . join ( path_wikisql , mode + '_tok.jsonl' ) if no_hs_tok : path_table = os . path . join ( path_wikisql , mode + '.tables.jsonl' ) else : path_table = os . path . join ( path_wikisql , mode + '_tok.tables.jsonl' ) data = [ ] table = { } with open ( path_sql ) as f : for idx , line in enumerate ( f ) : if toy_model and idx >= toy_size : break t1 = json . loads ( line . strip ( ) ) data . append ( t1 ) with open ( path_table ) as f : for idx , line in enumerate ( f ) : if toy_model and idx > toy_size : break t1 = json . loads ( line . strip ( ) ) table [ t1 [ 'id' ] ] = t1 return data , table