Update app.py

app.py

@@ -56,12 +56,12 @@ class ModelManager:
             logger.info("Starting model initialization...")
             model_name = "meta-llama/Llama-2-7b-chat-hf"
 
-            # Configure 4-bit quantization
+            # Configure 8-bit quantization instead of 4-bit
             bnb_config = BitsAndBytesConfig(
-                load_in_4bit=True,
-                bnb_4bit_use_double_quant=True,
-                bnb_4bit_quant_type="nf4",
-                bnb_4bit_compute_dtype=torch.bfloat16
+                load_in_8bit=True,
+                bnb_8bit_use_double_quant=True,
+                bnb_8bit_quant_type="nf8",
+                bnb_8bit_compute_dtype=torch.float16
             )
 
             # Load tokenizer with optimized settings
@@ -82,9 +82,6 @@ class ModelManager:
                 device_map="auto",
                 torch_dtype=torch.float16,
                 quantization_config=bnb_config,
-                use_flash_attention_2=True,
-                use_cache=True,
-                attn_implementation="flash_attention_2",
                 low_cpu_mem_usage=True,
             )