Update app.py

app.py

@@ -68,9 +68,9 @@ h3 {
 def initialize_model():
     """Initialize the model with appropriate configurations"""
     quantization_config = BitsAndBytesConfig(
-        load_in_8bit=True,
-        bnb_8bit_compute_dtype=torch.bfloat16,
-        bnb_8bit_use_double_quant=True
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_use_double_quant=True
     )
 
     tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
@@ -82,7 +82,8 @@ def initialize_model():
         torch_dtype=torch.float16,
         device_map="auto",
         attn_implementation="flash_attention_2",
-        quantization_config=quantization_config
+        quantization_config=quantization_config,
+        llm_int8_enable_fp32_cpu_offload=True
     )
 
     return model, tokenizer