Update app.py

app.py

@@ -1,25 +1,14 @@
 import os
-import sys
 import subprocess
 
-if not os.path.exists("ChatGLM-6b-onnx-u8s8"):
-    subprocess.run(["git", "lfs", "install"])
-    subprocess.run(["git", "clone", "https://huggingface.co/K024/ChatGLM-6b-onnx-u8s8"])
-    os.chdir("ChatGLM-6b-onnx-u8s8")
-    subprocess.run(["pip", "install", "-r", "requirements.txt"])
-    sys.path.append(os.getcwd())
-else:
-    sys.path.append(os.path.join(os.getcwd(), "ChatGLM-6b-onnx-u8s8"))
+if not os.path.exists("chatglm4-ggml-int4.bin"):
+    os.system("wget https://huggingface.co/npc0/chatglm-4-9b-int4/resolve/main/chatglm4-ggml-int4.bin")
 
-from model import ChatGLMModel#, chat_template
-
-model = ChatGLMModel()
+import chatglm_cpp
+pipeline = chatglm_cpp.Pipeline("chatglm4-ggml-int4.bin")
+# pipeline.chat([chatglm_cpp.ChatMessage(role="user", content="你好")])
 # history = []
 
-max_tokens = 2048
-temperature = 0.7
-top_p = 0.7
-top_k = 50
 prompt = """
 現在有些文本，文本詳細且複雜。 它包含細節,可以縮減和綜合為關鍵要點。 你的任務是提取最重要的概念,重點關注主要思路,提供一個概述而不失去精髓。 你的總結應該:
 
@@ -44,15 +33,13 @@ prompt = """
 
 def sum_chain_l1(text, p_bar):
     docs = []
-    for i in p_bar(range(len(text)//2000+1)):
-        t = text[i*2000:i*2000+2048]
+    for i in p_bar(range(len(text)//8000+1)):
+        t = text[i*8000:i*8000+8196]
         if len(t) > 0:
-            for answer in model.generate_iterate(prompt+t,
-                            max_generated_tokens=max_tokens,
-                            top_k=top_k,
-                            top_p=top_p,
-                            temperature=temperature):
-                yield f"{'='*8} {i+1}/{len(text)//2000+1} {'='*8}\n{answer}"
+            for answer in pipeline.stream_chat(
+                            chatglm_cpp.ChatMessage(role="user",
+                                                    content=prompt+t)):
+                yield f"{'='*8} {i+1}/{len(text)//8000+1} {'='*8}\n{answer}"
             docs.append(answer)
     yield docs
 
@@ -61,11 +48,9 @@ def sum_chain_l2_deprecated(docs, p_bar):
     i = 0
     for doc in p_bar(docs[1:]):
         i += 1
-        for answer in model.generate_iterate(prompt+"\n"+hist+"\n"+doc,
-                    max_generated_tokens=max_tokens,
-                    top_k=top_k,
-                    top_p=top_p,
-                    temperature=temperature):
+        for answer in pipeline.stream_chat(
+                            chatglm_cpp.ChatMessage(role="user",
+                                                    content=prompt+"\n"+hist+"\n"+doc)):
             yield f"{'='*8} {i}/{len(docs)} {'='*8}\n{answer}"
         hist = answer
     yield hist