Update app.py

app.py

@@ -29,7 +29,7 @@ def format_prompt(message, history):
     return prompt
 
 
-def chat_inf(system_prompt, prompt, history, temp, tokens, top_p, client_choice):
+def chat_inf(system_prompt, prompt, history, temp, tokens, top_p, seed, client_choice):
     client = clients[int(client_choice) - 1]
     if not history:
         history = []
@@ -38,17 +38,15 @@ def chat_inf(system_prompt, prompt, history, temp, tokens, top_p, client_choice)
         hist_len = len(history)
         print(hist_len)
 
-    generate_kwargs = dict(
-        temperature=temp,
-        max_new_tokens=tokens,
-        top_p=top_p,
-    )
+    sampler = keras_nlp.samplers.TopKSampler(k=5, seed=seed, top_p=top_p, temperature=temp)
+    client.compile(sampler=sampler)
+    
     formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
     stream = client.generate(formatted_prompt)
     output = ""
 
     for response in stream:
-        output += response
+        output += response.str(tokens)
         yield [(prompt, output)]
     history.append((prompt, output))
     yield history