follow KumaTea/KumaGLM

README.md

@@ -2,7 +2,7 @@
 title: KumaGLM Lite
 emoji: 🐨
 colorFrom: blue
-colorTo: blue
+colorTo: purple
 sdk: gradio
 sdk_version: 3.24.1
 app_file: app.py

app.py

@@ -0,0 +1,139 @@
+from fix_int8 import fix_pytorch_int8
+fix_pytorch_int8()
+
+
+# import subprocess
+# result = subprocess.run(['git', 'clone', 'https://huggingface.co/KumaTea/twitter-int8', 'model'], capture_output=True, text=True)
+# print(result.stdout)
+
+
+# Credit:
+# https://huggingface.co/spaces/ljsabc/Fujisaki/blob/main/app.py
+
+
+import torch
+import gradio as gr
+from transformers import AutoTokenizer, GenerationConfig, AutoModel
+
+
+# device = torch.device('cpu')
+# torch.cuda.current_device = lambda : device
+
+model = AutoModel.from_pretrained(
+    "KumaTea/twitter-int4",
+    trust_remote_code=True,
+    revision="e2aecb2"
+).float()  # .to(device)
+tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True, revision="4de8efe")
+
+# dump a log to ensure everything works well
+# print(model.peft_config)
+# We have to use full precision, as some tokens are >65535
+model.eval()
+# print(model)
+
+torch.set_default_tensor_type(torch.FloatTensor)
+
+
+def evaluate(context, temperature, top_p, top_k):
+    generation_config = GenerationConfig(
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        #repetition_penalty=1.1,
+        num_beams=1,
+        do_sample=True,
+    )
+    with torch.no_grad():
+        input_text = f"Context: {context}Answer: " 
+        ids = tokenizer.encode(input_text)
+        input_ids = torch.LongTensor([ids]).to('cpu')
+        out = model.generate(
+            input_ids=input_ids,
+            max_length=160,
+            generation_config=generation_config
+        )
+        out_text = tokenizer.decode(out[0]).split("Answer: ")[1]
+        return out_text
+
+
+def evaluate_stream(msg, history, temperature, top_p):
+    generation_config = GenerationConfig(
+        temperature=temperature,
+        top_p=top_p,
+        #repetition_penalty=1.1,
+        num_beams=1,
+        do_sample=True,
+    )
+
+    history.append([msg, None])
+
+    context = ""
+    if len(history) > 4:
+        history.pop(0)
+
+    for j in range(len(history)):
+        history[j][0] = history[j][0].replace("<br>", "")
+
+    # concatenate context
+    for h in history[:-1]:
+        context += h[0] + "||" + h[1] + "||"
+
+    context += history[-1][0]
+    context = context.replace(r'<br>', '')
+
+    # TODO: Avoid the tokens are too long.
+    CUTOFF = 224
+    while len(tokenizer.encode(context)) > CUTOFF:
+        # save 15 token size for the answer
+        context = context[15:]
+
+    h = []
+    print("History:", history)
+    print("Context:", context)
+    for response, h in model.stream_chat(tokenizer, context, h, max_length=CUTOFF, top_p=top_p, temperature=temperature):
+        history[-1][1] = response
+        yield history, ""
+
+    #return response
+
+
+title = """<h1 align="center">KumaGLM</h1>
+<h3 align='center'>这是一个 AI Kuma，你可以与他聊天，或者直接在文本框按下Enter</h3>
+<p align='center'>采用 INT4 量化，速度很慢，仅作备用</p>"""
+footer =  """<p align='center'>
+本项目基于
+<a href='https://github.com/ljsabc/Fujisaki' target='_blank'>ljsabc/Fujisaki</a>
+，模型采用
+<a href='https://huggingface.co/THUDM/chatglm-6b' target='_blank'>THUDM/chatglm-6b</a>
+。
+</p>
+<p align='center'>
+<em>每天起床第一句！</em>
+</p>"""
+
+with gr.Blocks() as demo:
+    gr.HTML(title)
+    state = gr.State()
+    with gr.Row():
+        with gr.Column(scale=2):
+            temp = gr.components.Slider(minimum=0, maximum=1.1, value=0.8, label="Temperature",
+                info="温度参数，越高的温度生成的内容越丰富，但是有可能出现语法问题。小的温度也能帮助生成更相关的回答。")
+            top_p = gr.components.Slider(minimum=0.5, maximum=1.0, value=0.975, label="Top-p",
+                info="top-p参数，只输出前p>top-p的文字，越大生成的内容越丰富，但也可能出现语法问题。数字越小似乎上下文的衔接性越好。")
+            #code = gr.Textbox(label="temp_output", info="解码器输出")
+            #top_k = gr.components.Slider(minimum=1, maximum=200, step=1, value=25, label="Top k",
+            #    info="top-k参数，下一个输出的文字会从top-k个文字中进行选择，越大生成的内容越丰富，但也可能出现语法问题。数字越小似乎上下文的衔接性越好。")
+            
+        with gr.Column(scale=3):
+            chatbot = gr.Chatbot(label="聊天框", info="")
+            msg = gr.Textbox(label="输入框", placeholder="最近过得怎么样？",
+                info="输入你的内容，按[Enter]发送。也可以什么都不填写生成随机数据。对话一般不能太长，否则就复读机了，建议清除数据。")
+            clear = gr.Button("清除聊天")
+
+    msg.submit(evaluate_stream, [msg, chatbot, temp, top_p], [chatbot, msg])
+    clear.click(lambda: None, None, chatbot, queue=False)
+    gr.HTML(footer)
+
+demo.queue()
+demo.launch(debug=False)

fix_int8.py

@@ -0,0 +1,29 @@
+import os
+import sys
+
+
+def fix_pytorch_int8():
+    valid_path = [p for p in sys.path if p and os.path.isdir(p)]
+
+    for path in valid_path:
+        for folder in os.listdir(path):
+            if 'torch' in folder:
+                packages_path = path
+                break
+
+    fix_path = f'{packages_path}/torch/nn/parameter.py'
+
+    with open(fix_path, 'r') as f:
+        text = f.read()
+
+    if 'if data.dtype == torch.int8' not in text:
+        text = text.replace(
+            '            return torch.Tensor._make_subclass(cls, data, requires_grad)', 
+            '            if data.dtype == torch.int8:\n' \
+            '                requires_grad = False\n' \
+            '            return torch.Tensor._make_subclass(cls, data, requires_grad)'
+        )
+        with open(fix_path, 'w') as f:
+            f.write(text)
+
+        return print('Fixed torch/nn/parameter.py')

requirements.txt

@@ -0,0 +1,20 @@
+# https://huggingface.co/spaces/ljsabc/Fujisaki/blob/main/requirements.txt
+
+# int8
+bitsandbytes>=0.37.1
+accelerate>=0.17.1
+
+# chatglm
+protobuf>=3.19.5,<4
+transformers>=4.27.1
+icetk
+cpm_kernels>=1.0.11
+
+# 
+datasets>=2.10.1
+git+https://github.com/huggingface/peft.git  # 最新版本 >=0.3.0.dev0
+
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.0.0+cpu
+torchvision==0.15.1+cpu
+torchaudio==2.0.1+cpu