Update

app.py

@@ -5,12 +5,16 @@ import re
 import gradio as gr
 import os
 from llama_cpp import Llama
-from gpt4all import GPT4All
+# from gpt4all import GPT4All
 import transformers
 # from transformers import GemmaTokenizer, AutoModelForCausalLM
 # from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 import accelerate
 import torch
+import subprocess
+
+subprocess.run('pip install llama-cpp-python==0.2.75 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124', shell=True)
+subprocess.run('pip install llama-cpp-agent==0.2.10', shell=True)
 
 # HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
@@ -71,48 +75,48 @@ def txt_to_html(text):
 
 def deidentify_doc(pdftext="", prompt="", maxtokens=600, temperature=1.2, top_probability=0.95):
 
-    # prompt = "Please anonymize the following clinical note. Replace all the following information with the term '[redacted]': Redact any strings that might be a name or initials, patients’ names, doctors’ names, the names Dr., redact any medical staff names, redact any strings that might be a location or address, such as '3970 Longview Drive', redact any strings that look like 'age 37', redact any dates and registration numbers, redact professions such as 'manager', redact any contact information."
+    prompt = "Please anonymize the following clinical note. Replace all the following information with the term '[redacted]': Redact any strings that might be a name or initials, patients’ names, doctors’ names, the names Dr., redact any medical staff names, redact any strings that might be a location or address, such as '3970 Longview Drive', redact any strings that look like 'age 37', redact any dates and registration numbers, redact professions such as 'manager', redact any contact information."
     
-    # output = model.create_chat_completion(
-    #                 messages = [
-    #                     {"role": "assistant", "content": prompt},
-    #                     {
-    #                         "role": "user",
-    #                         "content": pdftext
-    #                     }
-    #                 ],
-    #                 max_tokens=800,
-    #                 temperature=0
-    #             )
-    # output = output['choices'][0]['message']['content']
+    output = model.create_chat_completion(
+                    messages = [
+                        {"role": "assistant", "content": prompt},
+                        {
+                            "role": "user",
+                            "content": pdftext
+                        }
+                    ],
+                    max_tokens=800,
+                    temperature=0
+                )
+    output = output['choices'][0]['message']['content']
 
     # if (pdftext):
     #     prompt = prompt + ': ' + pdftext
     # output = model.generate(prompt=prompt, max_tokens=1024, n_batch=128)
 
-    messages = [
-        {"role": "assistant",
-         "content": prompt},
-        {"role": "user",
-         "content": pdftext}, ]
-    prompt = model.tokenizer.apply_chat_template(
-        messages,
-        tokenize=False,
-        add_generation_prompt=True
-    )
-    terminators = [
-        model.tokenizer.eos_token_id,
-        model.tokenizer.convert_tokens_to_ids("<|eot_id|>")
-    ]
-    outputs = model(
-        prompt,
-        max_new_tokens=maxtokens,
-        eos_token_id=terminators,
-        do_sample=True,
-        temperature=temperature,
-        top_p=top_probability,
-    )
-    output = outputs[0]["generated_text"][len(prompt):]
+    # messages = [
+    #     {"role": "assistant",
+    #      "content": prompt},
+    #     {"role": "user",
+    #      "content": pdftext}, ]
+    # prompt = model.tokenizer.apply_chat_template(
+    #     messages,
+    #     tokenize=False,
+    #     add_generation_prompt=True
+    # )
+    # terminators = [
+    #     model.tokenizer.eos_token_id,
+    #     model.tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    # ]
+    # outputs = model(
+    #     prompt,
+    #     max_new_tokens=maxtokens,
+    #     eos_token_id=terminators,
+    #     do_sample=True,
+    #     temperature=temperature,
+    #     top_p=top_probability,
+    # )
+    # output = outputs[0]["generated_text"][len(prompt):]
 
     return output
 
@@ -141,19 +145,19 @@ def pdf_to_text(files, output_folder, prompt, maxtokens=600, temperature=1.2, to
     display_text = "All selected reports are anonymized and results are saved in " + output_folder
     return anonymized_text
 
-# model_id = "D:/llama/meta-llama/Meta-Llama-3-8B-Instruct.Q5_K_M.gguf"
-# model = Llama(model_path=model_id, n_ctx=2048, n_threads=8, n_gpu_layers=32, n_batch=64)
+model_id = "Meta-Llama-3-8B-Instruct.Q5_K_M.gguf"
+model = Llama(model_path=model_id, n_ctx=2048, n_threads=8, n_gpu_layers=81, n_batch=64)
 
 # model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf", n_threads=8, device='gpu')
 # model.chat_session()
 
-model_id = "Meta-Llama-3-8B-Instruct"
-model = transformers.pipeline(
-    "text-generation",
-    model=model_id,
-    model_kwargs={"torch_dtype": torch.bfloat16},
-    device="cuda",
-)
+# model_id = "Meta-Llama-3-8B-Instruct"
+# model = transformers.pipeline(
+#     "text-generation",
+#     model=model_id,
+#     model_kwargs={"torch_dtype": torch.bfloat16},
+#     device="cuda",
+# )
 
 css = ".gradio-container {background: 'logo.png'}"
 temp_slider = gr.Slider(minimum=0, maximum=2, value=0.2, label="Temperature Value")