update model

app.py

@@ -30,6 +30,12 @@ import huggingface_hub
 import logging
 import sys
 from PIL import Image
+import gc
+
+def flush():
+  gc.collect()
+  torch.cuda.empty_cache()
+  torch.cuda.reset_peak_memory_stats()
 
 #Token do huggingface
 HF_TOKEN: Optional[str] = os.getenv("HF_TOKEN")    
@@ -54,7 +60,7 @@ for pasta in pastas:
 
 # Configuração do Streamlit
 st.sidebar.title("Configuração de LLM")
-sidebar_option = st.sidebar.radio("Selecione o LLM", ["Ollama", "OpenAI", "HF Local"])
+sidebar_option = st.sidebar.radio("Selecione o LLM", ["OpenAI", "HF Local"])
 # logo_url = 'app\logos\logo-sicoob.jpg'
 # st.sidebar.image(logo_url)
 import base64
@@ -73,12 +79,13 @@ with open("sicoob-logo.png", "rb") as f:
     )
 
 
-if sidebar_option == "Ollama":
-    Settings.llm = Ollama(model="llama3.2:latest", request_timeout=500.0, num_gpu=1)
-    Settings.embed_model = OllamaEmbedding(model_name="nomic-embed-text:latest")
-elif sidebar_option == "OpenAI":
+#if sidebar_option == "Ollama":
+   # Settings.llm = Ollama(model="llama3.2:latest", request_timeout=500.0, num_gpu=1)
+   # Settings.embed_model = OllamaEmbedding(model_name="nomic-embed-text:latest")
+if sidebar_option == "gpt-3.5":
     from llama_index.llms.openai import OpenAI
     from llama_index.embeddings.openai import OpenAIEmbedding
+    os.environ["OPENAI_API_KEY"] = "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"
     Settings.llm = OpenAI(model="gpt-3.5-turbo")
     Settings.embed_model = OpenAIEmbedding(model_name="text-embedding-ada-002")
 elif sidebar_option == 'HF Local':
@@ -86,41 +93,53 @@ elif sidebar_option == 'HF Local':
     logging.basicConfig(stream=sys.stdout, level=logging.INFO)
     logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
 
-    query_wrapper_prompt = PromptTemplate(
-     "Você é um assistente virtual capaz de interagir normalmente, além de"
-        " fornecer informações sobre organogramas e listar funcionários."
-        " Aqui estão os documentos relevantes para o contexto:\n"
-        "{context_str}"
-        "\nInstrução: Use o histórico da conversa anterior, ou o contexto acima, para responder."
-    )
+    #query_wrapper_prompt = PromptTemplate(
+    #"Below are several documents about a company "
+    #"Write a response that appropriately completes the request.\n\n"
+    #"### Instruction:\n{query_str}\n\n### Response:"
+    #)
     #Embedding do huggingface
     Settings.embed_model = HuggingFaceEmbedding(
     model_name="BAAI/bge-small-en-v1.5"
     )
-
     #Carregamento do modelo local, descomentar o modelo desejado
+    
     llm = HuggingFaceLLM(
-        # context_window=2048,
-        # max_new_tokens=256,
-        # generate_kwargs={"do_sample": False},
-        # query_wrapper_prompt=query_wrapper_prompt,
+        context_window=2048,
+        max_new_tokens=256,
+        generate_kwargs={"do_sample": False},
+        #query_wrapper_prompt=query_wrapper_prompt,
         #model_name="Qwen/Qwen2.5-Coder-32B-Instruct",
         #model_name="Qwen/Qwen2.5-14B-Instruct",
         # model_name="meta-llama/Llama-3.2-3B",
         #model_name="HuggingFaceH4/zephyr-7b-beta",
         # model_name="meta-llama/Meta-Llama-3-8B",
-        #model_name="numind/NuExtract-1.5",
-        model_name="meta-llama/Llama-3.2-3B",
+        model_name="numind/NuExtract-1.5",
+        #model_name="meta-llama/Llama-3.2-3B",
         
-        tokenizer_name="meta-llama/Llama-3.2-3B",
+        tokenizer_name="numind/NuExtract-1.5",
         device_map="auto",
-        # tokenizer_kwargs={"max_length": 2048},
+        tokenizer_kwargs={"max_length": 2048},
         # uncomment this if using CUDA to reduce memory usage
-        model_kwargs={"torch_dtype": torch.float16},
+        model_kwargs={"torch_dtype": torch.bfloat16},
     )
+    chat = [
+
+    {"role": "user", "content": "Hello, how are you?"},
+
+    {"role": "assistant", "content": "I'm doing great. How can I help you today?"},
+
+    {"role": "user", "content": "I'd like to show off how chat templating works!"},
+
+    ]
+
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained("numind/NuExtract-1.5")
 
+    tokenizer.apply_chat_template(chat, tokenize=False)
 
-    # Settings.chunk_size = 512
+    Settings.chunk_size = 512
     Settings.llm = llm
 
 else: