Update app.py

app.py

@@ -26,11 +26,6 @@ def process(action, base_model_id, dataset, system_prompt, user_prompt, schema):
         result = prompt_model(fine_tuned_model_id, system_prompt, user_prompt, schema)
     return result
 
-# Preprocess the dataset
-def preprocess(examples):
-    model_inputs = tokenizer(examples["text"], text_target=examples["sql"], max_length=512, truncation=True)
-    return model_inputs
-
 def fine_tune_model(base_model_id, dataset):
 #    tokenizer = download_model(base_model_id)
 #    fine_tuned_model_id = upload_model(base_model_id, tokenizer)
@@ -42,6 +37,11 @@ def fine_tune_model(base_model_id, dataset):
     model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct"
     model = AutoModelForCausalLM.from_pretrained(model_name)    
     tokenizer = AutoTokenizer.from_pretrained(model_name)    
+
+    # Preprocess the dataset
+    def preprocess(examples):
+        model_inputs = tokenizer(examples["text"], text_target=examples["sql"], max_length=512, truncation=True)
+        return model_inputs
     
     dataset = dataset.map(preprocess, batched=True)