random.seed(seedn)  
sample_ratio = 0.5  
data_path = '/home/rokabe/data2/cava/data/solid-state_dataset_2019-06-27_upd.json'  # path to the inorganic crystal synthesis data (json)  
data = json.load(open(data_path, 'r'))    
num_sample = int(len(data)*sample_ratio)    
separator=' || '    
cut = ';'    
rand_indices = random.sample(range(len(data)), num_sample)    
data1 = [data[i] for i in rand_indices]    
### dataset = Dataset_Ceq2Ope(data1, index=None, te_ratio=0.1, separator=separator, cut=cut).dataset   
hf_model = "distilbert-base-uncased"  
model_name = hf_usn + '/ope_bert_v1.2'# '/syn_distilgpt2_v2'  
tk_model = hf_model #"Dagobert42/gpt2-finetuned-material-synthesis"#'m3rg-iitd/matscibert'##hf_model # set tokenizer model loaded from HF (usually same as hf_model)  
load_pretrained=False   # If True, load the model from 'model_name'. Else, load the pre-trained model from hf_model.   
pad_tokenizer=False  
save_indices = True  
rm_ckpts = True  
{}
transformers pytorch distilbert fill-mask autotrain_compatible endpoints_compatible region:us