update. remote api focus

app.py

@@ -2,6 +2,11 @@ from flask import Flask, request, jsonify, send_from_directory
 import requests
 import os
 from dotenv import load_dotenv
+import traceback
+from PIL import Image
+from pdf2image import convert_from_bytes
+import base64
+from io import BytesIO
 
 load_dotenv()
 
@@ -9,18 +14,81 @@ app = Flask(__name__)
 
 API_URL = "https://api-inference.huggingface.co/models/"
 headers = {"Authorization": f"Bearer {os.getenv('HUGGINGFACE_API_KEY')}"}
-
-# Sample text for testing
-sample_text = """
-This is a sample text for testing our RAG chatbot. 
-It contains information about artificial intelligence and machine learning.
-AI and ML are revolutionizing various industries and improving efficiency.
-"""
+HF_REPO_ID = os.getenv('HF_REPO_ID')  # Your Hugging Face repo ID
 
 def query(payload, model):
     response = requests.post(API_URL + model, headers=headers, json=payload)
     return response.json()
 
+# Process PDFs using Hugging Face's PDF processing model
+def process_pdfs():
+    pdf_url = f"https://huggingface.co/spaces/{HF_REPO_ID}/resolve/main/data/your_pdf_file.pdf"
+    
+    try:
+        # Download PDF
+        pdf_response = requests.get(pdf_url)
+        pdf_response.raise_for_status()  # This will raise an exception for HTTP errors
+        
+        print(f"PDF downloaded successfully. Content length: {len(pdf_response.content)} bytes")
+        
+        # Convert PDF to images
+        images = convert_from_bytes(pdf_response.content)
+        print(f"Converted PDF to {len(images)} images")
+        
+        # Process each image
+        vision_model = "google/vit-base-patch16-224"
+        summaries = []
+        
+        for i, image in enumerate(images):
+            # Convert image to base64
+            buffered = BytesIO()
+            image.save(buffered, format="PNG")
+            img_str = base64.b64encode(buffered.getvalue()).decode()
+            
+            # Process image with vision model
+            payload = {
+                "inputs": [
+                    {
+                        "image": img_str,
+                        "text": "Describe the content of this image in detail."
+                    }
+                ]
+            }
+            response = query(payload, vision_model)
+            print(f"Page {i+1} processing response:", json.dumps(response, indent=2))
+            
+            if isinstance(response, list) and len(response) > 0 and 'generated_text' in response[0]:
+                summaries.append(response[0]['generated_text'])
+            else:
+                summaries.append(f"Error processing page {i+1}")
+        
+        return " ".join(summaries)
+    except Exception as e:
+        print(f"Error in process_pdfs: {str(e)}")
+        print(traceback.format_exc())
+        return f"Error processing PDF: {str(e)}"
+
+# Get the summary of PDFs
+pdf_summary = process_pdfs()
+print("PDF Summary:", pdf_summary)
+
+# Get embeddings for the summary
+embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
+if not pdf_summary.startswith("Error"):
+    try:
+        summary_embedding = query({"inputs": pdf_summary}, embedding_model)[0]
+        print("Successfully created summary embedding")
+    except Exception as e:
+        print(f"Error getting embedding: {str(e)}")
+        print(traceback.format_exc())
+        summary_embedding = None
+else:
+    print("Skipping embedding due to PDF processing error")
+    summary_embedding = None
+
+if summary_embedding is None:
+    print("WARNING: summary_embedding is None. The chatbot will not be able to provide meaningful responses.")
+
 @app.route('/')
 def home():
     return send_from_directory('.', 'index.html')
@@ -29,17 +97,17 @@ def home():
 def ask():
     prompt = request.json['question']
     
-    # Use sentence-transformers model for embedding
-    embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
-    context_embedding = query({"inputs": sample_text}, embedding_model)[0]
+    # Get embedding for the question
     query_embedding = query({"inputs": prompt}, embedding_model)[0]
     
-    # Simple dot product similarity
-    similarity = sum(a*b for a, b in zip(context_embedding, query_embedding))
+    # Calculate similarity
+    similarity = sum(a*b for a, b in zip(query_embedding, summary_embedding))
+    
+    # Generate character response
+    character_traits = "I am a knowledgeable and friendly AI assistant based on the content of the provided PDFs."
+    input_text = f"Character: {character_traits}\nContext: {pdf_summary}\nUser: {prompt}\nCharacter's response:"
     
-    # Generate response using T5 model
-    generator_model = "google/flan-t5-small"
-    input_text = f"Context: {sample_text}\n\nQuestion: {prompt}\n\nAnswer:"
+    generator_model = "google/flan-t5-base"
     response = query({"inputs": input_text}, generator_model)[0]["generated_text"]
 
     return jsonify({'response': response})

character_generator.py

@@ -1,10 +0,0 @@
-def generate_character_response(context, prompt):
-    # Define character traits based on your PDF content
-    character_traits = "I am a knowledgeable and friendly AI assistant based on the content of the provided PDF."
-    
-    input_text = f"Character: {character_traits}\nContext: {context}\nUser: {prompt}\nCharacter's response:"
-    
-    # Use your text generation model here
-    response = generator(input_text, max_length=150, num_return_sequences=1)[0]['generated_text']
-    
-    return response

model_loader.py

@@ -1,15 +0,0 @@
-from transformers import AutoTokenizer, AutoModel, pipeline
-from huggingface_hub import hf_hub_download
-
-def load_model():
-    tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
-    model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2", device_map="cpu")
-    return tokenizer, model
-
-def load_generator():
-    return pipeline('text2text-generation', model='google/flan-t5-base', device_map="cpu")
-
-def download_pdf():
-    # Replace 'your_pdf_file.pdf' with the actual name of your PDF file on Hugging Face
-    file_path = hf_hub_download(repo_id="your_username/your_repo_name", filename="your_pdf_file.pdf")
-    return file_path

pdf_processor.py

@@ -1,27 +0,0 @@
-import fitz  # PyMuPDF
-import numpy as np
-from transformers import AutoTokenizer, AutoModel
-import faiss
-
-def extract_text_from_pdf(file_path):
-    text = ""
-    with fitz.open(file_path) as doc:
-        for page in doc:
-            text += page.get_text()
-    return text
-
-def process_pdf(pdf_text):
-    chunks = [pdf_text[i:i+512] for i in range(0, len(pdf_text), 512)]
-    tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
-    model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
-    
-    embeddings = []
-    for chunk in chunks:
-        inputs = tokenizer(chunk, padding=True, truncation=True, max_length=512, return_tensors="pt")
-        outputs = model(**inputs)
-        embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy())
-    
-    embeddings = np.array(embeddings)
-    index = faiss.IndexFlatL2(embeddings.shape[1])
-    index.add(embeddings.astype('float32'))
-    return chunks, index

project_ouline.md

@@ -25,4 +25,13 @@ Deploy the project on Hugging Face Spaces
 Use static HTML for the frontend on Hugging Face Spaces
 Use Flask locally to test out features
 Focus on PDF functionality for now (VTT and JSON are stretch goals)
-Store the PDF file in a 'data/' folder within the project structure
+Store the PDF file in a 'data/' folder within the project structure
+PDF Storage: All PDF files should be stored in the Hugging Face repository, not locally.
+Model Hosting: All machine learning models (for embedding, text generation, etc.) should be hosted on Hugging Face, not run locally.
+Heavy Computations: All computationally intensive tasks such as PDF processing, text embedding, and response generation should be performed on Hugging Face's servers.
+API Usage: Interact with Hugging Face models and services via their API, sending requests from your local machine but having the processing done remotely.
+Local Testing:
+Flask Server: Run a local Flask server for development and testing purposes.
+Minimal Local Dependencies: Keep local dependencies to a minimum, mainly Flask and libraries needed for API interactions.
+Local Web Interface: Serve a simple HTML/JavaScript frontend locally for testing the chatbot interface.
+API Key Management: Use environment variables to manage API keys locally without exposing them.

requirements.txt

@@ -1,3 +1,4 @@
 flask
 requests
-python-dotenv
+python-dotenv
+PyMuPDF