Update app.py

app.py

@@ -4,23 +4,24 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 from threading import Thread
 
+torch.set_default_device("cuda")
+tokenizer = AutoTokenizer.from_pretrained(
+    "cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
+    trust_remote_code=True
+)
+model = AutoModelForCausalLM.from_pretrained(
+    "cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
+    torch_dtype="auto",
+    load_in_4bit=True,
+    trust_remote_code=True
+)
+
+system_prompt = "<|im_start|>system\nYou are Dolphin, a helpful AI assistant.<|im_end|>"
+
+
 @spaces.GPU(duration=120)
 def predict(message, history):
-    torch.set_default_device("cuda")
-
-    tokenizer = AutoTokenizer.from_pretrained(
-        "cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
-        trust_remote_code=True
-    )
-    model = AutoModelForCausalLM.from_pretrained(
-        "cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
-        torch_dtype="auto",
-        load_in_4bit=True,
-        trust_remote_code=True
-    )
     history_transformer_format = history + [[message, ""]]
-
-    system_prompt = "<|im_start|>system\nYou are Dolphin, a helpful AI assistant.<|im_end|>"
     messages = system_prompt + "".join(["".join(["\n<|im_start|>user\n" + item[0], "<|im_end|>\n<|im_start|>assistant\n" + item[1]]) for item in history_transformer_format])
     input_ids = tokenizer([messages], return_tensors="pt").to('cuda')
     streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)