app.py

import csv
import sys

# Increase CSV field size limit
csv.field_size_limit(sys.maxsize)



import gradio as gr
import pandas as pd

def data_pre_processing(file_responses):
    console_messages.append("Starting data pre-processing...")
    # Financial Weights can be anything (ultimately the row-wise weights are aggregated and the corresponding fractions are obtained from that rows' total tax payed)
    
    try: # Define the columns to be processed
        
        # Developing Numeric Columns
        # Convert columns to numeric and fill NaN values with 0
        file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'], errors='coerce').fillna(0)
        file_responses['Latest estimated Tax payment?'] = pd.to_numeric(file_responses['Latest estimated Tax payment?'], errors='coerce').fillna(0)
        
        # Adding a new column 'TotalWeightageAllocated' by summing specific columns by their names
        file_responses['TotalWeightageAllocated'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_3_TaxWeightageAllocated']

        

        # Creating Datasets (we assume everything has been provided to us in English, or the translations have been done already)
        # Renaming the datasets into similar column headings
        initial_dataset_1 = file_responses.rename(columns={
            'Personal_TaxDirection_1_Wish': 'Problem_Description',
            'Personal_TaxDirection_1_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_1_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]
        
        initial_dataset_2 = file_responses.rename(columns={
            'Personal_TaxDirection_2_Wish': 'Problem_Description',
            'Personal_TaxDirection_2_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_2_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]
        
        initial_dataset_3 = file_responses.rename(columns={
            'Personal_TaxDirection_3_Wish': 'Problem_Description',
            'Personal_TaxDirection_3_GeographicalLocation': 'Geographical_Location',
            'Personal_TaxDirection_3_TaxWeightageAllocated': 'Financial_Weight'
        })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']]


        
        # Calculating the actual TaxAmount to be allocated against each WISH (by overwriting the newly created columns)
        initial_dataset_1['Financial_Weight'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        initial_dataset_2['Financial_Weight'] = file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        initial_dataset_3['Financial_Weight'] = file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated']
        
        # Removing useless rows
        # Drop rows where Problem_Description is NaN or an empty string
        initial_dataset_1 = initial_dataset_1.dropna(subset=['Problem_Description'], axis=0)
        initial_dataset_2 = initial_dataset_2.dropna(subset=['Problem_Description'], axis=0)
        initial_dataset_3 = initial_dataset_3.dropna(subset=['Problem_Description'], axis=0)
        
        # Convert 'Problem_Description' column to string type
        initial_dataset_1['Problem_Description'] = initial_dataset_1['Problem_Description'].astype(str)
        initial_dataset_2['Problem_Description'] = initial_dataset_2['Problem_Description'].astype(str)
        initial_dataset_3['Problem_Description'] = initial_dataset_3['Problem_Description'].astype(str)
        
        # Merging the Datasets
        # Vertically concatenating (merging) the 3 DataFrames
        merged_dataset = pd.concat([initial_dataset_1, initial_dataset_2, initial_dataset_3], ignore_index=True)

        
        # Different return can be used to check the processing
        console_messages.append("Data pre-processing completed.")
        # return file_responses
        return merged_dataset
        
    except Exception as e:
        console_messages.append(f"Error during data pre-processing: {str(e)}")
        # return str(e), console_messages
        return None






import spacy
from transformers import AutoTokenizer, AutoModel
import torch

# Load SpaCy model
# Install the 'en_core_web_sm' model if it isn't already installed
try:
    nlp = spacy.load('en_core_web_sm')
except OSError:
    # Instead of this try~catch, we could also include this < https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0.tar.gz > in the requirements.txt to directly load it
    from spacy.cli import download
    download('en_core_web_sm')
    nlp = spacy.load('en_core_web_sm')


# Load Hugging Face Transformers model
tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")




import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
# Download necessary NLTK data
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')




import numpy as np
import sentencepiece as sp
from transformers import pipeline
# Load a summarization model
summarizer = pipeline("summarization")

def Summarized_text(passed_text):
    try:
        # Summarization
        summarize_text = summarizer(passed_text, max_length=70, min_length=30, do_sample=False)[0]['summary_text']
        return summarize_text
    except Exception as e:
        print(f"Summarization failed: {e}")
        return passed_text
    ###### Will uncomment Summarization during final deployment... as it takes a lot of time
    
def Lemmatize_text(text):
    # Text Cleaning
    text = re.sub(r'[^\w\s]', '', text)
    text = re.sub(r'\d+', '', text)
    text = re.sub(r'http\S+', '', text)  # Remove https URLs
    text = re.sub(r'www\.\S+', '', text)  # Remove www URLs
    
    # Tokenize and remove stopwords
    tokens = word_tokenize(text.lower())
    stop_words = set(stopwords.words('english'))
    custom_stopwords = {'example', 'another'}  # Add custom stopwords
    tokens = [word for word in tokens if word not in stop_words and word not in custom_stopwords]

    # NER - Remove named entities
    doc = nlp(' '.join(tokens))
    tokens = [token.text for token in doc if not token.ent_type_]
    
    # POS Tagging (optional)
    pos_tags = nltk.pos_tag(tokens)
    tokens = [word for word, pos in pos_tags if pos in ['NN', 'NNS']]  # Filter nouns
    
    # Lemmatize tokens using SpaCy
    doc = nlp(' '.join(tokens))
    lemmatized_text = ' '.join([token.lemma_ for token in doc])

    return lemmatized_text  # Return the cleaned and lemmatized text


from random import random
def text_processing_for_domain(text):
    # First, get the summarized text
    summarized_text = ""
    # summarized_text = Summarized_text(text)
    
    # Then, lemmatize the original text
    lemmatized_text = ""
    lemmatized_text = Lemmatize_text(text)

    if lemmatized_text and summarized_text:
        # Join both the summarized and lemmatized text
        if random() > 0.5:
            combined_text = summarized_text + "  " + lemmatized_text
        else:
            combined_text = lemmatized_text + "  " + summarized_text
        return combined_text
    elif summarized_text:
        return summarized_text
    elif lemmatized_text:
        return lemmatized_text
    else:
        return "Sustainability and Longevity" # Default FailSafe
    


from sentence_transformers import SentenceTransformer
from sklearn.cluster import AgglomerativeClustering, KMeans
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import silhouette_score
from bertopic import BERTopic
from collections import Counter


def extract_problem_domains(df, 
                            text_column='Processed_ProblemDescription_forDomainExtraction',
                            cluster_range=(5, 15), 
                            top_words=10):
    console_messages.append("Extracting Problem Domains...")
    
    # Sentence Transformers approach
    model = SentenceTransformer('all-mpnet-base-v2')
    embeddings = model.encode(df[text_column].tolist())
    
    # Perform hierarchical clustering with Silhouette Analysis
    silhouette_scores = []
    for n_clusters in range(cluster_range[0], cluster_range[1] + 1):
        clustering = AgglomerativeClustering(n_clusters=n_clusters)
        cluster_labels = clustering.fit_predict(embeddings)
        silhouette_avg = silhouette_score(embeddings, cluster_labels)
        silhouette_scores.append(silhouette_avg)
    
    # Determine the optimal number of clusters
    optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores))
    
    # Perform clustering with the optimal number of clusters
    clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters)
    cluster_labels = clustering.fit_predict(embeddings)
    
    # Get representative words for each cluster
    cluster_representations = {}
    for i in range(optimal_n_clusters):
        cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split()
        cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)]
    
    # Map cluster labels to representative words
    df["Problem_Cluster"] = cluster_labels
    df['Problem_Category_Words'] = [cluster_representations[label] for label in cluster_labels]
    
    # console_messages.append("Returning from Problem Domain Extraction function.")
    console_messages.append("Problem Domain Extraction completed.")    
    return df, optimal_n_clusters













import spacy
from geopy.geocoders import Nominatim
from geopy.exc import GeocoderTimedOut, GeocoderUnavailable
import pandas as pd

nlp = spacy.load('en_core_web_sm')
geolocator = Nominatim(user_agent="my_agent")

def extract_and_geocode_locations(text, user_locations):
    # Extract locations from text
    doc = nlp(text)
    extracted_locations = [ent.text for ent in doc.ents if ent.label_ in ['GPE', 'LOC']]
    
    # Combine extracted locations with user-provided locations
    all_locations = list(set(extracted_locations + user_locations.split(', ')))
    
    geocoded_locations = []
    for loc in all_locations:
        try:
            location = geolocator.geocode(loc)
            if location:
                geocoded_locations.append({
                    'name': loc,
                    'latitude': location.latitude,
                    'longitude': location.longitude,
                    'country': location.raw.get('display_name', '').split(', ')[-1]
                })
            else:
                # If geocoding fails, add the location without coordinates
                geocoded_locations.append({
                    'name': loc,
                    'latitude': None,
                    'longitude': None,
                    'country': None
                })
        except (GeocoderTimedOut, GeocoderUnavailable):
            print(f"Geocoding failed for {loc}")
            # Add the location without coordinates
            geocoded_locations.append({
                'name': loc,
                'latitude': None,
                'longitude': None,
                'country': None
            })
    
    return geocoded_locations

def text_processing_for_location(row):
    locations = extract_and_geocode_locations(row['Problem_Description'], row['Geographical_Location'])
    location_text = ' '.join([loc['name'] for loc in locations])
    processed_text = Lemmatize_text(location_text)
    return processed_text, locations

def extract_location_clusters(df, 
                              text_column='Processed_LocationText_forClustering', 
                              cluster_range=(3, 10), 
                              top_words=5):
    console_messages.append("Extracting Location Clusters...")
    
    # Sentence Transformers approach for embeddings
    model = SentenceTransformer('all-mpnet-base-v2')
    embeddings = model.encode(df[text_column].tolist())
    
    # Perform hierarchical clustering with Silhouette Analysis
    silhouette_scores = []
    for n_clusters in range(cluster_range[0], cluster_range[1] + 1):
        clustering = AgglomerativeClustering(n_clusters=n_clusters)
        cluster_labels = clustering.fit_predict(embeddings)
        silhouette_avg = silhouette_score(embeddings, cluster_labels)
        silhouette_scores.append(silhouette_avg)
    
    # Determine the optimal number of clusters
    optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores))
    
    # Perform clustering with the optimal number of clusters
    clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters)
    cluster_labels = clustering.fit_predict(embeddings)
    
    # Get representative words for each cluster
    cluster_representations = {}
    for i in range(optimal_n_clusters):
        cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split()
        cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)]
    
    # Map cluster labels to representative words
    df["Location_Cluster"] = cluster_labels
    df['Location_Category_Words'] = [cluster_representations[label] for label in cluster_labels]
    
    console_messages.append("Location Clustering completed.")    
    return df, optimal_n_clusters

















# def Extract_Location(text):
#     doc = nlp(text)
#     locations = [ent.text for ent in doc.ents if ent.label_ in ['GPE', 'LOC']]
#     return ' '.join(locations)

# def text_processing_for_location(text):
#     # Extract locations
#     locations_text = Extract_Location(text)
    
#     # Perform further text cleaning if necessary
#     processed_locations_text = Lemmatize_text(locations_text)
    
#     # Remove special characters, digits, and punctuation
#     processed_locations_text = re.sub(r'[^a-zA-Z\s]', '', processed_locations_text)
    
#     # Tokenize and remove stopwords
#     tokens = word_tokenize(processed_locations_text.lower())
#     stop_words = set(stopwords.words('english'))
#     tokens = [word for word in tokens if word not in stop_words]
    
#     # Join location words into a single string
#     final_locations_text = ' '.join(tokens)
    
#     return final_locations_text if final_locations_text else "India"
    

# def extract_location_clusters(df, 
#                               text_column='Processed_LocationText_forClustering', 
#                               cluster_range=(3, 10), 
#                               top_words=5):
#     console_messages.append("Extracting Location Clusters...")
    
#     # Sentence Transformers approach for embeddings
#     model = SentenceTransformer('all-mpnet-base-v2')
#     embeddings = model.encode(df[text_column].tolist())
    
#     # Perform hierarchical clustering with Silhouette Analysis
#     silhouette_scores = []
#     for n_clusters in range(cluster_range[0], cluster_range[1] + 1):
#         clustering = AgglomerativeClustering(n_clusters=n_clusters)
#         cluster_labels = clustering.fit_predict(embeddings)
#         silhouette_avg = silhouette_score(embeddings, cluster_labels)
#         silhouette_scores.append(silhouette_avg)
    
#     # Determine the optimal number of clusters
#     optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores))
    
#     # Perform clustering with the optimal number of clusters
#     clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters)
#     cluster_labels = clustering.fit_predict(embeddings)
    
#     # Get representative words for each cluster
#     cluster_representations = {}
#     for i in range(optimal_n_clusters):
#         cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split()
#         cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)]
    
#     # Map cluster labels to representative words
#     df["Location_Cluster"] = cluster_labels
#     df['Location_Category_Words'] = [cluster_representations[label] for label in cluster_labels]
    
#     console_messages.append("Location Clustering completed.")    
#     return df, optimal_n_clusters



















def nlp_pipeline(original_df):
    console_messages.append("Starting NLP pipeline...")
    
    # Data Preprocessing
    processed_df = data_pre_processing(original_df) # merged_dataset

    # Starting the Pipeline for Domain Extraction
    console_messages.append("Executing Text processing function for Domain identification")
    # Apply the text_processing_for_domain function to the DataFrame
    processed_df['Processed_ProblemDescription_forDomainExtraction'] = processed_df['Problem_Description'].apply(text_processing_for_domain)
    
    console_messages.append("Removing entries which could not be allocated to any Problem Domain")
    # processed_df = processed_df.dropna(subset=['Processed_ProblemDescription_forDomainExtraction'], axis=0)
    # Drop rows where 'Processed_ProblemDescription_forDomainExtraction' contains empty arrays
    processed_df = processed_df[processed_df['Processed_ProblemDescription_forDomainExtraction'].apply(lambda x: len(x) > 0)]
    
    # Domain Clustering
    try:
        processed_df, optimal_n_clusters = extract_problem_domains(processed_df)
        console_messages.append(f"Optimal clusters for Domain extraction: {optimal_n_clusters}")
    except Exception as e:
        console_messages.append(f"Error in extract_problem_domains: {str(e)}")
    console_messages.append("NLP pipeline for Problem Domain extraction completed.")

    
    console_messages.append("Starting NLP pipeline for Location extraction with text processing.")
    
    # Apply the text_processing_for_location function to the DataFrame
    # processed_df['Processed_LocationText_forClustering'] = processed_df['Problem_Description'].apply(text_processing_for_location)
    processed_df['Processed_LocationText_forClustering'], processed_df['Extracted_Locations'] = zip(*processed_df.apply(text_processing_for_location, axis=1))
        
    # Location Clustering
    try:
        processed_df, optimal_n_clusters = extract_location_clusters(processed_df)
        console_messages.append(f"Optimal clusters for Location extraction: {optimal_n_clusters}")
    except Exception as e:
        console_messages.append(f"Error in extract_location_clusters: {str(e)}")
    console_messages.append("NLP pipeline for location extraction completed.")

    
    console_messages.append("NLP pipeline completed.")
    return processed_df
    
        
    
    
    
    
console_messages = []
def process_excel(file):
    console_messages.append("Processing starts. Reading the uploaded Excel file...")
    # Ensure the file path is correct
    file_path = file.name if hasattr(file, 'name') else file
    # Read the Excel file
    df = pd.read_excel(file_path)

    try:
        # Process the DataFrame
        console_messages.append("Processing the DataFrame...")
        result_df = nlp_pipeline(df)

        # output_file = "Output_ProjectProposals.xlsx"
        output_file = "Output_Proposals.xlsx"
        result_df.to_excel(output_file, index=False)

        console_messages.append("Processing completed. Ready for download.")
        return output_file, "\n".join(console_messages)  # Return the processed DataFrame as Excel file
        
    except Exception as e:
        # return str(e)  # Return the error message
        # error_message = f"Error processing file: {str(e)}"
        # print(error_message)  # Log the error
        console_messages.append(f"Error during processing: {str(e)}")
        # return error_message, "Santanu Banerjee" # Return the error message to the user
        return None, "\n".join(console_messages)


        




# example_files = ['#TaxDirection (Responses)_BasicExample.xlsx',
#                  '#TaxDirection (Responses)_IntermediateExample.xlsx',
#                  '#TaxDirection (Responses)_UltimateExample.xlsx'
#                 ]

example_files = ['#TaxDirection (Responses)_BasicExample.xlsx',
                 '#TaxDirection (Responses)_IntermediateExample.xlsx',
                 ]


import random
a_random_object = random.choice(["&rArr;", "&rarrtl;", "&Rarr;", "&rarr;"])


# Define the Gradio interface
interface = gr.Interface(
    fn=process_excel,  # The function to process the uploaded file
    inputs=gr.File(type="filepath", label="Upload Excel File here. \t Be sure to check that the column headings in your upload are the same as in the Example files below. \t (Otherwise there will be Error during the processing)"),  # File upload input
    
    examples=example_files,  # Add the example files
    
    
    outputs=[
        gr.File(label="Download the processed Excel File containing the ** Project Proposals ** for each Location~Problem paired combination"),  # File download output
        gr.Textbox(label="Console Messages", lines=15, interactive=False)  # Console messages output
        ],
    
    
    # title="Excel File Uploader",
    # title="Upload Excel file containing #TaxDirections &rarr; Download HyperLocal Project Proposals\n",
    
    title = (
        "<p style='font-weight: bold; font-size: 25px; text-align: center;'>"
        "<span style='color: blue;'>Upload Excel file containing #TaxDirections</span> "
        
        # "<span style='color: brown; font-size: 35px;'>&rarr; </span>"
        # "<span style='color: brown; font-size: 35px;'>&rArr;  &rarrtl; &Rarr; </span>"
        "<span style='color: brown; font-size: 35px;'> " +a_random_object +" </span>"
        
        "<span style='color: green;'>Download HyperLocal Project Proposals</span>"
        "</p>\n"
    ),



    description=(
        "<p style='font-size: 12px; color: gray; text-align: center'>This tool allows for the systematic evaluation and proposal of solutions tailored to specific location-problem pairs, ensuring efficient resource allocation and project planning. For more information, visit <a href='https://santanban.github.io/TaxDirection/' target='_blank'>#TaxDirection weblink</a>.</p>"
        
        "<p style='font-weight: bold; font-size: 16px; color: blue;'>Upload an Excel file to process and download the result or use the Example files:</p>"
        "<p style='font-weight: bold; font-size: 15px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>"
        
        "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Processed output contains a Project Proposal for each Location~Problem paired combination (i.e. each cell).</p>"
        "<p style='font-weight: bold; font-size: 13px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>"

        
        "<p style='font-size: 12px; color: gray; text-align: center'>Note: The example files provided above are for demonstration purposes. Feel free to upload your own Excel files to see the results. If you have any questions, refer to the documentation-links or contact <a href='https://www.change.org/p/democracy-evolution-ensuring-humanity-s-eternal-existence-through-taxdirection' target='_blank'>support</a>.</p>"
        
    )  # Solid description with right-aligned second sentence

)



# Launch the interface
if __name__ == "__main__":
    interface.launch()