Update app.py

app.py

@@ -5,6 +5,7 @@ import re
 import gradio as gr
 from llama_cpp import Llama
 import os
+from gpt4all import GPT4All
 
 def process_document(pdf_path, page_ids=None):
 
@@ -63,25 +64,28 @@ def txt_to_html(text):
 
 def deidentify_doc(pdftext=""):
 
-    # prompt = "Please anonymize the following clinical note. Replace all the following information with the term '[redacted]': Redact any strings that might be a name or acronym or initials, patients’ names, doctors’ names, the names of the M.D. or Dr., redact any medical staff names, redact any strings that might be a location or address, such as '3970 Longview Drive', redact any strings that look like 'something years old' or 'age 37', redact any dates and IDs and record dates, redact professions such as 'manager', redact any contact information."
     prompt = "Please anonymize the following clinical note. Replace all the following information with the term '[redacted]': Redact any strings that might be a name or initials, patients’ names, doctors’ names, the names Dr., redact any medical staff names, redact any strings that might be a location or address, such as '3970 Longview Drive', redact any strings that look like 'age 37', redact any dates and registration numbers, redact professions such as 'manager', redact any contact information."
 
-    print('Input prompt is ',prompt)
-    print('Input pdf text is ',pdftext)
-
-    output = model.create_chat_completion(
-                    messages = [
-                        {"role": "assistant", "content": prompt},
-                        {
-                            "role": "user",
-                            "content": pdftext
-                        }
-                    ],
-                    max_tokens=600,
-                    temperature=0
-                )
-    output = output['choices'][0]['message']['content']
-    
+    # print('Input prompt is ',prompt)
+    # print('Input pdf text is ',pdftext)
+
+    # output = model.create_chat_completion(
+    #                 messages = [
+    #                     {"role": "assistant", "content": prompt},
+    #                     {
+    #                         "role": "user",
+    #                         "content": pdftext
+    #                     }
+    #                 ],
+    #                 max_tokens=600,
+    #                 temperature=0
+    #             )
+    # output = output['choices'][0]['message']['content']
+
+    if (pdftext):
+        prompt = prompt + ': ' + pdftext
+    output = model.generate(prompt=prompt, max_tokens=1024, n_batch=128)
+
     return output
 
 def pdf_to_text(file):
@@ -95,8 +99,11 @@ def pdf_to_text(file):
         file.write(html)
     return html
 
-model_id = "Meta-Llama-3-8B-Instruct.Q5_K_M.gguf"
-model = Llama(model_path=model_id, n_ctx=2048, n_threads=8, n_gpu_layers=20, n_batch=128)
+# model_id = "Meta-Llama-3-8B-Instruct.Q5_K_M.gguf"
+# model = Llama(model_path=model_id, n_ctx=2048, n_threads=8, n_gpu_layers=20, n_batch=128)
+
+model = GPT4All("Meta-Llama-3-8B-Instruct.Q4_0.gguf", n_threads=8, device='gpu')
+model.chat_session()
 
 css = ".gradio-container {background: 'logo.png'}"