Upload 2 files

app.py

@@ -0,0 +1,210 @@
+import streamlit as st
+import pandas as pd
+from functions import *
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def initialize_session_state():
+    if 'processing_complete' not in st.session_state:
+        st.session_state['processing_complete'] = False
+    if 'results_df' not in st.session_state:
+        st.session_state['results_df'] = None
+    if 'output_choice' not in st.session_state:
+        st.session_state['output_choice'] = "Download CSV"
+
+initialize_session_state()
+
+def main():
+    st.title("InfoSynth")
+    
+    df = None
+    
+    # File upload section
+    st.header("1. Upload Your Data")
+    data_source = st.radio("Choose a data source:", ["CSV File", "Google Sheet"])
+
+    if data_source == "CSV File":
+        uploaded_file = st.file_uploader("Choose a CSV file", type=['csv'])
+        
+        if uploaded_file is not None:
+            df = pd.read_csv(uploaded_file)
+    else:
+        st.info(
+        "Before proceeding, ensure your Google Sheet is shared with the service account. "
+        "You can find the service account email in your credentials.json file."
+        )
+        spreadsheet_id = st.text_input(
+            "Enter Google Spreadsheet ID", 
+            help="You can find this in the spreadsheet URL between /d/ and /edit"
+        )
+        
+        sheet_names = None
+        if spreadsheet_id:
+            try:
+                sheet_names = get_all_sheet_names(spreadsheet_id)
+                if not sheet_names:
+                    st.error("No sheets found in this spreadsheet. Please check the ID and permissions.")
+            except ValueError as e:
+                st.error(f"Error accessing spreadsheet: {str(e)}")
+                st.info("Please check the ID and permissions.")
+            except Exception as e:
+                st.error(f"Error accessing spreadsheet: {str(e)}")
+                sheet_names = []
+        
+        sheet_name = None
+        if sheet_names:
+            sheet_name = st.selectbox(
+                "Select Sheet Name",
+                options=sheet_names,
+                help="The name of the specific sheet to read from"
+            )
+        
+        if spreadsheet_id and sheet_name:
+            try:
+                df = load_google_sheet(spreadsheet_id, sheet_name)
+                if df is None or df.empty:
+                    st.error("No data found in the selected sheet.")
+            except Exception as e:
+                st.error(f"Error loading sheet data: {str(e)}")
+                df = None
+
+    if df is not None:
+        try:
+            # Display available columns for selection
+            st.header("2. Select Primary Column")
+            primary_column = st.selectbox(
+                "Choose the main column for analysis:",
+                options=df.columns.tolist()
+            )
+            
+            # Show data preview
+            st.header("3. Data Preview")
+            st.write("First 5 rows of your data:")
+            st.dataframe(df.head())            
+            
+            # Add Query Template Section
+            st.header("4. Query Template")
+            st.write(f"""
+            Create your query template using {primary_column} as a placeholder.
+            Example: "What products does {primary_column} offer?"
+            """)
+            
+            query_template = st.text_area(
+                "Enter your query template:",
+                value=f"Tell me about {{{primary_column}}}",
+                help=f"Use {{{primary_column}}} as a placeholder"
+            )
+            
+            # Preview generated queries
+            #if st.button("Preview Generated Queries"):
+            #    st.subheader("Generated Queries Preview")
+            #    # Get first 5 values from the selected column
+            #    sample_values = df[primary_column].head()
+            #    
+            #    # Display example queries
+            #    for value in sample_values:
+            #        generated_query = query_template.replace(
+            #            f"{{{primary_column}}}", str(value)
+            #        )
+            #        st.write(f"- {generated_query}")
+            #    
+            #    # Show total number of queries that will be generated
+            #    st.info(f"Total queries to be generated: {len(df)}")
+            
+            # Add confirmation and processing section
+            st.header("5. Process Queries")
+            total_queries = len(df[primary_column])
+            estimated_time = total_queries * 2  # 2 second per query due to rate limiting
+            
+            st.warning(f"""
+            ⚠️ Please confirm:
+            - Number of queries to process: {total_queries}
+            - Estimated processing time: {estimated_time} seconds ({estimated_time/60:.1f} minutes)
+            - This will use {total_queries} API calls
+            """)
+            
+            # Show sample of what will be processed
+            #st.subheader("Sample of data to be processed:")
+            #sample_df = df[[primary_column]].head()
+            #st.dataframe(sample_df)
+            
+            # Process button with confirmation
+            if st.button("Start Processing"):
+                with st.spinner("Processing queries..."):
+                    # Add a progress bar
+                    progress_bar = st.progress(0)
+                    
+                    results = []
+                    llm = setup_llm()
+                    for index, row in df.iterrows():
+                        try:
+                            value = row[primary_column]
+                            
+                            # Handle empty/null values
+                            if pd.isna(value) or str(value).strip() == '':
+                                results.append({
+                                    'input_value': value,
+                                    'result': 'NA'
+                                })
+                                continue
+                            
+                            query = query_template.replace(f"{{{primary_column}}}", str(value))
+                            
+                            # Display current processing item
+                            st.text(f"Processing: {value}")
+                            
+                            # Process query
+                            result = process_queries(pd.DataFrame([row]), primary_column, query)
+                            output = process_with_ai(result, query, llm)
+                            
+                            results.append({
+                                'input_value': value,
+                                'result': output.content
+                            })
+                            
+                            # Update progress
+                            progress_bar.progress((index + 1) / total_queries)
+                            
+                        except Exception as e:
+                            st.error(f"Error processing {value}: {str(e)}")
+                            continue
+                    
+                    # Show completion and results
+                    st.session_state['processing_complete'] = True
+                    st.session_state['results_df'] = pd.DataFrame(results, columns=['input_value', 'result'])
+
+            # Show results and save options if processing is complete
+            if st.session_state['processing_complete']:
+                st.success(f"✅ Completed processing {len(st.session_state['results_df'])} queries!")
+                
+                st.subheader("Results Preview:")
+                st.dataframe(st.session_state['results_df'].head())
+                
+                st.header("6. Save Results")
+                output_choice = st.radio("Choose an output format:", ["Download CSV", "Update Google Sheet"])
+
+                if output_choice == "Download CSV":
+                    csv = st.session_state['results_df'].to_csv(index=False)
+                    if st.download_button(
+                        "Download Complete Results (CSV)",
+                        csv,
+                        "search_results.csv",
+                        "text/csv",
+                        key='download-csv'
+                    ):
+                        st.success("✅ File downloaded successfully!")
+                        
+                elif output_choice == "Update Google Sheet":
+                    update_button = st.button("Confirm Update to Google Sheet")
+                    if update_button:
+                        try:
+                            write_to_google_sheet(spreadsheet_id, sheet_name, st.session_state['results_df'])
+                            st.success("✅ Results successfully added as new column!")
+                        except Exception as e:
+                            st.error(f"Error updating sheet: {str(e)}")
+        except Exception as e:
+            st.error(f"Error processing the file: {str(e)}")
+
+if __name__ == "__main__":
+    main()

functions.py

@@ -0,0 +1,162 @@
+import streamlit as st
+import pandas as pd
+import time
+from typing import List, Dict
+from serpapi import GoogleSearch
+from langchain_groq import ChatGroq
+from langchain.prompts import PromptTemplate
+import gspread
+from google.oauth2.service_account import Credentials
+import pandas as pd
+import os
+
+def get_sheet_client():
+    """Helper function to create authenticated Google Sheets client"""
+    try:
+        scope = ["https://www.googleapis.com/auth/spreadsheets"]
+        creds = Credentials.from_service_account_file("credentials.json", scopes=scope)
+        client = gspread.authorize(creds)
+        
+        # Get service account email for error messages
+        service_account_email = creds.service_account_email
+        st.session_state['service_account_email'] = service_account_email
+        
+        return client
+    except FileNotFoundError:
+        raise ValueError(
+            "credentials.json file not found. Please ensure it exists in the project directory."
+        )
+    except Exception as e:
+        raise ValueError(f"Error setting up Google Sheets client: {str(e)}")
+
+def get_worksheet(sheet_id: str, range_name: str = None):
+    """Helper function to get worksheet with improved error handling"""
+    try:
+        client = get_sheet_client()
+        sheet = client.open_by_key(sheet_id)
+        return sheet.worksheet(range_name) if range_name else sheet
+    except gspread.exceptions.SpreadsheetNotFound:
+        service_email = st.session_state.get('service_account_email', 'the service account')
+        raise ValueError(
+            f"Spreadsheet not found. Please verify:\n"
+            f"1. The spreadsheet ID is correct\n"
+            f"2. The sheet is shared with {service_email}\n"
+            f"3. Sharing permissions allow edit access"
+        )
+    except gspread.exceptions.WorksheetNotFound:
+        raise ValueError(f"Worksheet '{range_name}' not found in the spreadsheet")
+    except gspread.exceptions.APIError as e:
+        if 'PERMISSION_DENIED' in str(e):
+            service_email = st.session_state.get('service_account_email', 'the service account')
+            raise ValueError(
+                f"Permission denied. Please share the spreadsheet with {service_email} "
+                f"and ensure it has edit access."
+            )
+        raise ValueError(f"Google Sheets API error: {str(e)}")
+
+def process_queries(df: pd.DataFrame, primary_column: str, query_template: str) -> List[Dict]:
+    results = []
+    
+    serpapi_key = os.getenv("SERPAPI_API_KEY")  
+    for index, row in df.iterrows():
+        try:
+            value = row[primary_column]
+            query = query_template.replace(f"{{{primary_column}}}", str(value))
+            
+            # Perform search
+            search = GoogleSearch({
+                "q": query,
+                "gl": "in",
+                "api_key": serpapi_key,
+                "num": 5  
+            })
+            search_results = search.get_dict()
+            
+            # Store results
+            results.append({
+                primary_column: value,
+                "query": query,
+                "search_results": search_results.get("organic_results", [])
+            })
+            
+            # Rate limiting
+            time.sleep(1) 
+            
+            
+            if index % 10 == 0:  
+                st.write(f"Processed {index + 1} queries...")
+                
+        except Exception as e:
+            st.warning(f"Error processing query for {value}: {str(e)}")
+            continue
+    
+    return results
+
+def setup_llm():
+    """Setup LangChain with Groq"""
+    api_key=os.getenv("GROQ_API_KEY")
+    llm = ChatGroq(
+        api_key=api_key, 
+        model="llama-3.1-8b-instant",
+        temperature=0,
+        max_tokens=None,
+        timeout=None,
+        max_retries=2,
+    )
+    return llm
+
+def process_with_ai(search_results: dict, query: str, llm) -> str:
+    template = """
+    Extract ONLY the specific information requested from the search results for: {query}
+    
+    Search Results:
+    {search_results}
+    
+    Provide ONLY the extracted information as a simple text response. 
+    If multiple items exist, separate them with semicolons.
+    If no relevant information is found, respond with "Not found".
+    
+    For example:
+    - If asked for locations: "Bengaluru; Mumbai; Delhi"
+    - If asked for email: "[email protected]"
+    - If asked for address: "123 Main Street, City, Country"
+    """
+    
+    prompt = PromptTemplate(
+        input_variables=["query", "search_results"],
+        template=template
+    )
+    
+    chain = prompt | llm
+    response = chain.invoke({"query": query, "search_results": search_results})
+    
+    return response
+
+
+def load_google_sheet(sheet_id: str, range_name: str) -> pd.DataFrame:    
+    worksheet = get_worksheet(sheet_id,range_name)
+    data = worksheet.get_all_records()    
+    return pd.DataFrame(data)
+
+
+def write_to_google_sheet(sheet_id: str, range_name: str, results_df: pd.DataFrame):
+
+    worksheet = get_worksheet(sheet_id, range_name)
+    
+    all_values = worksheet.get_all_values()
+    num_rows = len(all_values)
+    next_col_num = len(all_values[0]) + 1
+    next_col_letter = chr(64 + next_col_num) 
+    
+    range = f'{next_col_letter}1:{next_col_letter}{num_rows}'
+    
+    values = [['AI Results']] + [[str(result)] for result in results_df['result']]
+    
+    worksheet.update(values, f'{range}')
+
+
+def get_all_sheet_names(sheet_id: str) -> List[str]:
+    
+    worksheet = get_worksheet(sheet_id)
+    sheets = map(lambda x: x.title, worksheet.worksheets())
+    return list(sheets)