Spaces:
Sleeping
Sleeping
| import csv | |
| import sys | |
| # Increase CSV field size limit | |
| csv.field_size_limit(sys.maxsize) | |
| import gradio as gr | |
| import pandas as pd | |
| def data_pre_processing(file_responses): | |
| consoleMessage_and_Print("Starting data pre-processing...") | |
| # Financial Weights can be anything (ultimately the row-wise weights are aggregated and the corresponding fractions are obtained from that rows' total tax payed) | |
| try: # Define the columns to be processed | |
| # Developing Numeric Columns | |
| # Convert columns to numeric and fill NaN values with 0 | |
| file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Latest estimated Tax payment?'] = pd.to_numeric(file_responses['Latest estimated Tax payment?'], errors='coerce').fillna(0) | |
| # Adding a new column 'TotalWeightageAllocated' by summing specific columns by their names | |
| file_responses['TotalWeightageAllocated'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] | |
| # Creating Datasets (we assume everything has been provided to us in English, or the translations have been done already) | |
| # Renaming the datasets into similar column headings | |
| initial_dataset_1 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_1_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_1_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_1_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| initial_dataset_2 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_2_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_2_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_2_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| initial_dataset_3 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_3_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_3_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_3_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| # Calculating the actual TaxAmount to be allocated against each WISH (by overwriting the newly created columns) | |
| initial_dataset_1['Financial_Weight'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| initial_dataset_2['Financial_Weight'] = file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| initial_dataset_3['Financial_Weight'] = file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| # Removing useless rows # Drop rows where Problem_Description is NaN or an empty string | |
| initial_dataset_1 = initial_dataset_1.dropna(subset=['Problem_Description'], axis=0) | |
| initial_dataset_2 = initial_dataset_2.dropna(subset=['Problem_Description'], axis=0) | |
| initial_dataset_3 = initial_dataset_3.dropna(subset=['Problem_Description'], axis=0) | |
| # Convert 'Problem_Description' column to string type | |
| initial_dataset_1['Problem_Description'] = initial_dataset_1['Problem_Description'].astype(str) | |
| initial_dataset_2['Problem_Description'] = initial_dataset_2['Problem_Description'].astype(str) | |
| initial_dataset_3['Problem_Description'] = initial_dataset_3['Problem_Description'].astype(str) | |
| # Merging the Datasets # Vertically concatenating (merging) the 3 DataFrames | |
| merged_dataset = pd.concat([initial_dataset_1, initial_dataset_2, initial_dataset_3], ignore_index=True) | |
| # Different return can be used to check the processing | |
| consoleMessage_and_Print("Data pre-processing completed.") | |
| return merged_dataset | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error during data pre-processing: {str(e)}") | |
| return None | |
| import spacy | |
| from transformers import AutoTokenizer, AutoModel | |
| import torch | |
| # Load SpaCy model | |
| # Install the 'en_core_web_sm' model if it isn't already installed | |
| try: | |
| nlp = spacy.load('en_core_web_sm') | |
| except OSError: | |
| # Instead of this try~catch, we could also include this < https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0.tar.gz > in the requirements.txt to directly load it | |
| from spacy.cli import download | |
| download('en_core_web_sm') | |
| nlp = spacy.load('en_core_web_sm') | |
| # Load Hugging Face Transformers model | |
| tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2") | |
| model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2") | |
| import re | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.tokenize import word_tokenize | |
| # Download necessary NLTK data | |
| nltk.download('punkt') | |
| nltk.download('stopwords') | |
| nltk.download('averaged_perceptron_tagger') | |
| import numpy as np | |
| import sentencepiece as sp | |
| from transformers import pipeline | |
| # Load a summarization model | |
| summarizer = pipeline("summarization") | |
| def Summarized_text(passed_text): | |
| try: | |
| # Summarization | |
| summarize_text = summarizer(passed_text, max_length=70, min_length=30, do_sample=False)[0]['summary_text'] | |
| return summarize_text | |
| except Exception as e: | |
| print(f"Summarization failed: {e}") | |
| return passed_text | |
| ###### Will uncomment Summarization during final deployment... as it takes a lot of time | |
| def Lemmatize_text(text): | |
| # Text Cleaning | |
| text = re.sub(r'[^\w\s]', '', text) | |
| text = re.sub(r'\d+', '', text) | |
| text = re.sub(r'http\S+', '', text) # Remove https URLs | |
| text = re.sub(r'www\.\S+', '', text) # Remove www URLs | |
| # Tokenize and remove stopwords | |
| tokens = word_tokenize(text.lower()) | |
| stop_words = set(stopwords.words('english')) | |
| custom_stopwords = {'example', 'another'} # Add custom stopwords | |
| tokens = [word for word in tokens if word not in stop_words and word not in custom_stopwords] | |
| # NER - Remove named entities | |
| doc = nlp(' '.join(tokens)) | |
| tokens = [token.text for token in doc if not token.ent_type_] | |
| # POS Tagging (optional) | |
| pos_tags = nltk.pos_tag(tokens) | |
| tokens = [word for word, pos in pos_tags if pos in ['NN', 'NNS']] # Filter nouns | |
| # Lemmatize tokens using SpaCy | |
| doc = nlp(' '.join(tokens)) | |
| lemmatized_text = ' '.join([token.lemma_ for token in doc]) | |
| return lemmatized_text # Return the cleaned and lemmatized text | |
| from random import random | |
| def text_processing_for_domain(text): | |
| # First, get the summarized text | |
| summarized_text = "" | |
| # summarized_text = Summarized_text(text) | |
| # Then, lemmatize the original text | |
| lemmatized_text = "" | |
| lemmatized_text = Lemmatize_text(text) | |
| if lemmatized_text and summarized_text: | |
| # Join both the summarized and lemmatized text | |
| if random() > 0.5: | |
| combined_text = summarized_text + " " + lemmatized_text | |
| else: | |
| combined_text = lemmatized_text + " " + summarized_text | |
| return combined_text | |
| elif summarized_text: | |
| return summarized_text | |
| elif lemmatized_text: | |
| return lemmatized_text | |
| else: | |
| return "Sustainability and Longevity" # Default FailSafe | |
| from sentence_transformers import SentenceTransformer | |
| from sklearn.cluster import AgglomerativeClustering, KMeans | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics import silhouette_score | |
| from bertopic import BERTopic | |
| from collections import Counter | |
| def extract_problem_domains(df, | |
| text_column='Processed_ProblemDescription_forDomainExtraction', | |
| cluster_range=(2, 10), | |
| top_words=10): | |
| consoleMessage_and_Print("Extracting Problem Domains...") | |
| # Sentence Transformers approach | |
| model = SentenceTransformer('all-mpnet-base-v2') | |
| embeddings = model.encode(df[text_column].tolist()) | |
| # Perform hierarchical clustering with Silhouette Analysis | |
| silhouette_scores = [] | |
| for n_clusters in range(cluster_range[0], cluster_range[1] + 1): | |
| clustering = AgglomerativeClustering(n_clusters=n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| silhouette_avg = silhouette_score(embeddings, cluster_labels) | |
| silhouette_scores.append(silhouette_avg) | |
| # Determine the optimal number of clusters | |
| optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores)) | |
| # Perform clustering with the optimal number of clusters | |
| clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| # Get representative words for each cluster | |
| cluster_representations = {} | |
| for i in range(optimal_n_clusters): | |
| cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split() | |
| cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)] | |
| # Map cluster labels to representative words | |
| df["Problem_Cluster"] = cluster_labels | |
| df['Problem_Category_Words'] = [cluster_representations[label] for label in cluster_labels] | |
| consoleMessage_and_Print("Problem Domain Extraction completed. Returning from Problem Domain Extraction function.") | |
| return df, optimal_n_clusters, cluster_representations | |
| def Extract_Location(text): | |
| doc = nlp(text) | |
| locations = [ent.text for ent in doc.ents if ent.label_ in ['GPE', 'LOC']] | |
| return ' '.join(locations) | |
| def text_processing_for_location(text): | |
| # Extract locations | |
| locations_text = Extract_Location(text) | |
| # Perform further text cleaning if necessary | |
| processed_locations_text = Lemmatize_text(locations_text) | |
| # Remove special characters, digits, and punctuation | |
| processed_locations_text = re.sub(r'[^a-zA-Z\s]', '', processed_locations_text) | |
| # Tokenize and remove stopwords | |
| tokens = word_tokenize(processed_locations_text.lower()) | |
| stop_words = set(stopwords.words('english')) | |
| tokens = [word for word in tokens if word not in stop_words] | |
| # Join location words into a single string | |
| final_locations_text = ' '.join(tokens) | |
| return final_locations_text if final_locations_text else "India" | |
| def extract_location_clusters(df, | |
| text_column1='Processed_LocationText_forClustering', # Extracted through NLP | |
| text_column2='Geographical_Location', # User Input | |
| cluster_range=(2, 10), | |
| top_words=10): | |
| # Combine the two text columns | |
| text_column = "Combined_Location_Text" | |
| df[text_column] = df[text_column1] + ' ' + df[text_column2] | |
| consoleMessage_and_Print("Extracting Location Clusters...") | |
| # Sentence Transformers approach for embeddings | |
| model = SentenceTransformer('all-mpnet-base-v2') | |
| embeddings = model.encode(df[text_column].tolist()) | |
| # Perform hierarchical clustering with Silhouette Analysis | |
| silhouette_scores = [] | |
| for n_clusters in range(cluster_range[0], cluster_range[1] + 1): | |
| clustering = AgglomerativeClustering(n_clusters=n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| silhouette_avg = silhouette_score(embeddings, cluster_labels) | |
| silhouette_scores.append(silhouette_avg) | |
| # Determine the optimal number of clusters | |
| optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores)) | |
| # Perform clustering with the optimal number of clusters | |
| clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| # Get representative words for each cluster | |
| cluster_representations = {} | |
| for i in range(optimal_n_clusters): | |
| cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split() | |
| cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)] | |
| # Map cluster labels to representative words | |
| df["Location_Cluster"] = cluster_labels | |
| df['Location_Category_Words'] = [cluster_representations[label] for label in cluster_labels] | |
| df = df.drop(text_column, axis=1) | |
| consoleMessage_and_Print("Location Clustering completed.") | |
| return df, optimal_n_clusters, cluster_representations | |
| def create_cluster_dataframes(processed_df): | |
| # Create a dataframe for Financial Weights | |
| budget_cluster_df = processed_df.pivot_table( | |
| values='Financial_Weight', | |
| index='Location_Cluster', | |
| columns='Problem_Cluster', | |
| aggfunc='sum', | |
| fill_value=0) | |
| # Create a dataframe for Problem Descriptions | |
| problem_cluster_df = processed_df.groupby(['Location_Cluster', 'Problem_Cluster'])['Problem_Description'].apply(list).unstack() | |
| return budget_cluster_df, problem_cluster_df | |
| from random import uniform | |
| from transformers import GPTNeoForCausalLM, GPT2Tokenizer | |
| def generate_project_proposal(prompt): # Generate the proposal | |
| default_proposal = "Hyper-local Sustainability Projects would lead to Longevity of the self and Prosperity of the community. Therefore UNSDGs coupled with Longevity initiatives should be focused upon." | |
| # model_Name = "EleutherAI/gpt-neo-2.7B" | |
| # tempareCHUR = uniform(0.3,0.6) | |
| model_Name = "EleutherAI/gpt-neo-1.3B" | |
| tempareCHUR = uniform(0.5,0.8) | |
| consoleMessage_and_Print(f"Trying to access {model_Name} model. The Prompt is: \n{prompt}") | |
| model = GPTNeoForCausalLM.from_pretrained(model_Name) | |
| tokenizer = GPT2Tokenizer.from_pretrained(model_Name) | |
| model_max_token_limit = 2000 #2048 #1500 | |
| try: | |
| # input_ids = tokenizer.encode(prompt, return_tensors="pt") | |
| # Truncate the prompt to fit within the model's input limits | |
| # Adjust as per your model's limit | |
| input_ids = tokenizer.encode(prompt, return_tensors="pt", truncation=True, max_length = int(2*model_max_token_limit/3) ) | |
| print("Input IDs shape:", input_ids.shape) | |
| input_length = input_ids.shape[1] # Slice off the input part if the input length is known | |
| pad_tokenId = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id # Padding with EOS token may always be great | |
| attentionMask = input_ids.ne(pad_tokenId).long() | |
| # Generate the output | |
| output = model.generate( | |
| input_ids, | |
| min_length = int(model_max_token_limit/7), # minimum length of the generated output | |
| max_new_tokens = int(model_max_token_limit/3), | |
| num_return_sequences=1, | |
| no_repeat_ngram_size=2, | |
| temperature=tempareCHUR, | |
| attention_mask=attentionMask, # This was previously not being used | |
| pad_token_id=pad_tokenId | |
| ) | |
| print("Output shape:", output.shape) | |
| # Decode the output to text | |
| full_returned_segment = tokenizer.decode(output[0], skip_special_tokens=True) | |
| PP_in_fullReturn = "Project Proposal:" in full_returned_segment | |
| if output is not None and output.shape[1] > 0: | |
| # Decode the output | |
| if output.shape[1] > input_length and PP_in_fullReturn: | |
| generated_part = tokenizer.decode(output[0][input_length:], skip_special_tokens=True) | |
| else: | |
| generated_part = tokenizer.decode(output[0], skip_special_tokens=True) | |
| else: | |
| # Handle the error case, e.g., return an empty string or a default value | |
| raise Exception("Error generating proposal: output is empty or None") | |
| proposal = generated_part.strip() | |
| # if "Project Proposal:" in proposal: | |
| # proposal = proposal.split("Project Proposal:", 1)[1].strip() | |
| print("Generated Proposal: \n", proposal,"\n\n") | |
| return proposal | |
| except Exception as e: | |
| print("Error generating proposal:", str(e)) | |
| return default_proposal | |
| import copy | |
| def create_project_proposals(budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters): | |
| consoleMessage_and_Print("\n Starting function: create_project_proposals") | |
| proposals = {} | |
| for loc in budget_cluster_df.index: | |
| consoleMessage_and_Print(f"\n loc: {loc}") | |
| for prob in budget_cluster_df.columns: | |
| consoleMessage_and_Print(f"\n prob: {prob}") | |
| location = ", ".join([item.strip() for item in location_clusters[loc] if item]) # Clean and join | |
| problem_domain = ", ".join([item.strip() for item in problem_clusters[prob] if item]) # Clean and join | |
| shuffled_descriptions = copy.deepcopy(problem_cluster_df.loc[loc, prob]) | |
| # Create a deep copy of the problem descriptions, shuffle it, and join the first 10 | |
| print("location: ", location) | |
| print("problem_domain: ", problem_domain) | |
| print("problem_descriptions: ", shuffled_descriptions) | |
| # Check if problem_descriptions is valid (not NaN and not an empty list) | |
| if isinstance(shuffled_descriptions, list) and shuffled_descriptions: | |
| # print(f"\nGenerating proposal for location: {location}, problem domain: {problem_domain}") | |
| consoleMessage_and_Print(f"Generating PP") | |
| random.shuffle(shuffled_descriptions) | |
| # Prepare the prompt | |
| # problems_summary = "; \n".join(shuffled_descriptions[:3]) # Limit to first 3 for brevity | |
| # problems_summary = "; \n".join([f"Problem: {desc}" for desc in shuffled_descriptions[:5]]) | |
| problems_summary = "; \n".join([f"Problem {i+1}: {desc}" for i, desc in enumerate(shuffled_descriptions[:7])]) | |
| # problems_summary = "; \n".join(shuffled_descriptions) # Join all problem descriptions | |
| # prompt = f"Generate a solution oriented project proposal for the following:\n\nLocation: {location}\nProblem Domain: {problem_domain}\nProblems: {problems_summary}\n\nProject Proposal:" | |
| # prompt = f"Generate a solution-oriented project proposal for the following public problem (only output the proposal):\n\n Geographical/Digital Location: {location}\nProblem Category: {problem_domain}\nProblems: {problems_summary}\n\nProject Proposal:" | |
| # prompt = f"Generate a singular solution-oriented project proposal bespoke to the following Location~Domain cluster of public problems:\n\n Geographical/Digital Location: {location}\nProblem Domain: {problem_domain}\nProblems: {problems_summary}\n\nProject Proposal: \t" | |
| prompt = f"Generate a singular solution-oriented project proposal bespoke to the following Location~Domain cluster of public problems:\n\n Geographical/Digital Location: {location}\nProblem Domain: {problem_domain}\n\n {problems_summary}\n\nSingle Combined Project Proposal: \t" | |
| proposal = generate_project_proposal(prompt) | |
| # Check if proposal is valid | |
| if isinstance(proposal, str) and proposal.strip(): # Valid string that's not empty | |
| proposals[(loc, prob)] = proposal | |
| else: | |
| print(f"Skipping empty problem descriptions for location: {location}, problem domain: {problem_domain}") | |
| return proposals | |
| # def create_project_proposals(budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters): | |
| # print("\n Starting function: create_project_proposals") | |
| # console_messages.append("\n Starting function: create_project_proposals") | |
| # proposals = {} | |
| # for loc in budget_cluster_df.index: | |
| # print("\n loc: ", loc) | |
| # console_messages.append(f"\n loc: {loc}") | |
| # for prob in budget_cluster_df.columns: | |
| # console_messages.append(f"\n prob: {prob}") | |
| # print("\n prob: ", prob) | |
| # location = ", ".join([item.strip() for item in location_clusters[loc] if item]) # Clean and join | |
| # problem_domain = ", ".join([item.strip() for item in problem_clusters[prob] if item]) # Clean and join | |
| # problem_descriptions = problem_cluster_df.loc[loc, prob] | |
| # print("location: ",location) | |
| # print("problem_domain: ",problem_domain) | |
| # print("problem_descriptions: ",problem_descriptions) | |
| # if problem_descriptions:# and not pd.isna(problem_descriptions): | |
| # print(f"\nGenerating proposal for location: {location}, problem domain: {problem_domain}") | |
| # # console_messages.append(f"\nGenerating proposal for location: {location}, problem domain: {problem_domain}") | |
| # # Prepare the prompt | |
| # problems_summary = "; \n".join(problem_descriptions[:3]) # Limit to first 3 for brevity | |
| # # problems_summary = "; ".join(problem_descriptions) | |
| # # prompt = f"Generate a project proposal for the following:\n\nLocation: {location}\nProblem Domain: {problem_domain}\nProblems: {problems_summary}\nBudget: ${financial_weight:.2f}\n\nProject Proposal:" | |
| # prompt = f"Generate a solution oriented project proposal for the following:\n\nLocation: {location}\nProblem Domain: {problem_domain}\nProblems: {problems_summary}\n\nProject Proposal:" | |
| # proposal = generate_project_proposal(prompt) | |
| # proposals[(loc, prob)] = proposal | |
| # print("Generated Proposal: ", proposal) | |
| # else: | |
| # print(f"Skipping empty problem descriptions for location: {location}, problem domain: {problem_domain}") | |
| # return proposals | |
| # def create_project_proposals(budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters): | |
| # print("\n Starting function: create_project_proposals") | |
| # console_messages.append("\n Starting function: create_project_proposals") | |
| # proposals = {} | |
| # for loc in budget_cluster_df.index: | |
| # for prob in budget_cluster_df.columns: | |
| # location = ", ".join(location_clusters[loc]) | |
| # problem_domain = ", ".join(problem_clusters[prob]) | |
| # problem_descriptions = problem_cluster_df.loc[loc, prob] | |
| # if problem_descriptions: | |
| # proposal = generate_project_proposal( | |
| # problem_descriptions, | |
| # location, | |
| # problem_domain) | |
| # proposals[(loc, prob)] = proposal | |
| # console_messages.append("\n Exiting function: create_project_proposals") | |
| # return proposals | |
| def nlp_pipeline(original_df): | |
| consoleMessage_and_Print("Starting NLP pipeline...") | |
| # Data Preprocessing | |
| processed_df = data_pre_processing(original_df) # merged_dataset | |
| # Starting the Pipeline for Domain Extraction | |
| consoleMessage_and_Print("Executing Text processing function for Domain identification") | |
| # Apply the text_processing_for_domain function to the DataFrame | |
| processed_df['Processed_ProblemDescription_forDomainExtraction'] = processed_df['Problem_Description'].apply(text_processing_for_domain) | |
| consoleMessage_and_Print("Removing entries which could not be allocated to any Problem Domain") | |
| # processed_df = processed_df.dropna(subset=['Processed_ProblemDescription_forDomainExtraction'], axis=0) | |
| # Drop rows where 'Processed_ProblemDescription_forDomainExtraction' contains empty arrays | |
| processed_df = processed_df[processed_df['Processed_ProblemDescription_forDomainExtraction'].apply(lambda x: len(x) > 0)] | |
| # Domain Clustering | |
| try: | |
| processed_df, optimal_n_clusters, problem_clusters = extract_problem_domains(processed_df) | |
| consoleMessage_and_Print(f"Optimal clusters for Domain extraction: {optimal_n_clusters}") | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error in extract_problem_domains: {str(e)}") | |
| consoleMessage_and_Print("NLP pipeline for Problem Domain extraction completed.") | |
| consoleMessage_and_Print("Starting NLP pipeline for Location extraction with text processing.") | |
| # Apply the text_processing_for_location function to the DataFrame | |
| processed_df['Processed_LocationText_forClustering'] = processed_df['Problem_Description'].apply(text_processing_for_location) | |
| # processed_df['Processed_LocationText_forClustering'], processed_df['Extracted_Locations'] = zip(*processed_df.apply(text_processing_for_location, axis=1)) | |
| # Location Clustering | |
| try: | |
| processed_df, optimal_n_clusters, location_clusters = extract_location_clusters(processed_df) | |
| consoleMessage_and_Print(f"Optimal clusters for Location extraction: {optimal_n_clusters}") | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error in extract_location_clusters: {str(e)}") | |
| consoleMessage_and_Print("NLP pipeline for location extraction completed.") | |
| # Create cluster dataframes | |
| budget_cluster_df, problem_cluster_df = create_cluster_dataframes(processed_df) | |
| print("Clustering Done...") | |
| # return processed_df, budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters | |
| print("\n location_clusters: ", location_clusters) | |
| print("\n problem_clusters: ", problem_clusters) | |
| # # Generate project proposals | |
| # location_clusters = dict(enumerate(processed_df['Location_Category_Words'].unique())) | |
| # problem_clusters = dict(enumerate(processed_df['Problem_Category_Words'].unique())) | |
| # print("\n location_clusters_2: ", location_clusters) | |
| # print("\n problem_clusters_2: ", problem_clusters) | |
| project_proposals = create_project_proposals(budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters) | |
| consoleMessage_and_Print("NLP pipeline completed.") | |
| return processed_df, budget_cluster_df, problem_cluster_df, project_proposals, location_clusters, problem_clusters | |
| console_messages = [] | |
| def consoleMessage_and_Print(some_text = ""): | |
| console_messages.append(some_text) | |
| print(some_text) | |
| def process_excel(file): | |
| consoleMessage_and_Print("Processing starts. Reading the uploaded Excel file...") | |
| # Ensure the file path is correct | |
| file_path = file.name if hasattr(file, 'name') else file | |
| # Read the Excel file | |
| df = pd.read_excel(file_path) | |
| try: | |
| # Process the DataFrame | |
| consoleMessage_and_Print("Processing the DataFrame...") | |
| processed_df, budget_cluster_df, problem_cluster_df, project_proposals, location_clusters, problem_clusters = nlp_pipeline(df) | |
| # processed_df, budget_cluster_df, problem_cluster_df, location_clusters, problem_clusters = nlp_pipeline(df) | |
| consoleMessage_and_Print("Error was here") | |
| #This code first converts the dictionary to a DataFrame with a single column for the composite key. | |
| #Then, it splits the composite key into separate columns for Location_Cluster and Problem_Cluster. | |
| #Finally, it reorders the columns and writes the DataFrame to an Excel sheet. | |
| try: # Meta AI Solution | |
| # Convert project_proposals dictionary to DataFrame | |
| project_proposals_df = pd.DataFrame(list(project_proposals.items()), columns=['Location_Cluster_Problem_Cluster', 'Solutions Proposed']) | |
| # consoleMessage_and_Print("CheckPoint 1") | |
| # Split the composite key into separate columns | |
| project_proposals_df[['Location_Cluster', 'Problem_Cluster']] = project_proposals_df['Location_Cluster_Problem_Cluster'].apply(pd.Series) | |
| # consoleMessage_and_Print("CheckPoint 2") | |
| # Drop the composite key column | |
| project_proposals_df.drop('Location_Cluster_Problem_Cluster', axis=1, inplace=True) | |
| # consoleMessage_and_Print("CheckPoint 3") | |
| # Reorder the columns | |
| project_proposals_df = project_proposals_df[['Location_Cluster', 'Problem_Cluster', 'Solutions Proposed']] | |
| # consoleMessage_and_Print("CheckPoint 4") | |
| except Exception as e: | |
| consoleMessage_and_Print("Meta AI Solution did not work, trying CHATGPT solution") | |
| try: | |
| # Convert project_proposals dictionary to DataFrame | |
| project_proposals_df = pd.DataFrame.from_dict( | |
| proposals, orient='index', columns=['Solutions Proposed'] | |
| ) | |
| # If the index is a tuple, it automatically becomes a MultiIndex, so we handle naming correctly: | |
| if isinstance(project_proposals_df.index, pd.MultiIndex): | |
| project_proposals_df.index.names = ['Location_Cluster', 'Problem_Cluster'] | |
| else: | |
| # If for some reason it's not a MultiIndex, we name it appropriately | |
| project_proposals_df.index.name = 'Cluster' | |
| # Reset index to have Location_Cluster and Problem_Cluster as columns | |
| project_proposals_df.reset_index(inplace=True) | |
| except Exception as e: | |
| print(e) | |
| # ### Convert project_proposals dictionary to DataFrame | |
| # project_proposals_df = pd.DataFrame.from_dict(project_proposals, orient='index', columns=['Solutions Proposed']) | |
| # project_proposals_df.index.names = ['Location_Cluster', 'Problem_Cluster'] | |
| # project_proposals_df.reset_index(inplace=True) | |
| consoleMessage_and_Print("Creating the Excel file.") | |
| output_filename = "OutPut_PPs.xlsx" | |
| with pd.ExcelWriter(output_filename) as writer: | |
| processed_df.to_excel(writer, sheet_name='Input_Processed', index=False) | |
| budget_cluster_df.to_excel(writer, sheet_name='Financial_Weights') | |
| problem_cluster_df.to_excel(writer, sheet_name='Problem_Descriptions') | |
| try: | |
| project_proposals_df.to_excel(writer, sheet_name='Project_Proposals', index=False) | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error during Project Proposal excelling at the end: {e}") | |
| try: | |
| location_clusters_df = pd.DataFrame({'Cluster_Id': list(location_clusters.keys()), | |
| 'Location_Cluster': list(location_clusters.values())}) | |
| location_clusters_df.to_excel(writer, sheet_name='Location_Clusters', index=False) | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error during Location Cluster Dataframing: {e}") | |
| try: | |
| problem_clusters_df = pd.DataFrame({'Cluster_Id': list(problem_clusters.keys()), | |
| 'Problem_Cluster': list(problem_clusters.values())}) | |
| problem_clusters_df.to_excel(writer, sheet_name='Problem_Clusters', index=False) | |
| except Exception as e: | |
| consoleMessage_and_Print(f"Error during Problem Cluster Dataframing: {e}") | |
| # # Ensure location_clusters and problem_clusters are in DataFrame format | |
| # if isinstance(location_clusters, pd.DataFrame): | |
| # location_clusters.to_excel(writer, sheet_name='Location_Clusters', index=False) | |
| # else: | |
| # consoleMessage_and_Print("Converting Location Clusters to df") | |
| # pd.DataFrame(location_clusters).to_excel(writer, sheet_name='Location_Clusters', index=False) | |
| # if isinstance(problem_clusters, pd.DataFrame): | |
| # problem_clusters.to_excel(writer, sheet_name='Problem_Clusters', index=False) | |
| # else: | |
| # consoleMessage_and_Print("Converting Problem Clusters to df") | |
| # pd.DataFrame(problem_clusters).to_excel(writer, sheet_name='Problem_Clusters', index=False) | |
| consoleMessage_and_Print("Processing completed. Ready for download.") | |
| return output_filename, "\n".join(console_messages) # Return the processed DataFrame as Excel file | |
| except Exception as e: | |
| # return str(e) # Return the error message | |
| # error_message = f"Error processing file: {str(e)}" | |
| # print(error_message) # Log the error | |
| consoleMessage_and_Print(f"Error during processing: {str(e)}") | |
| # return error_message, "Santanu Banerjee" # Return the error message to the user | |
| return None, "\n".join(console_messages) | |
| example_files = [] | |
| example_files.append('#TaxDirection (Responses)_BasicExample.xlsx') | |
| # example_files.append('#TaxDirection (Responses)_IntermediateExample.xlsx') | |
| # example_files.append('#TaxDirection (Responses)_UltimateExample.xlsx') | |
| import random | |
| a_random_object = random.choice(["⇒", "↣", "↠", "→"]) | |
| # Define the Gradio interface | |
| interface = gr.Interface( | |
| fn=process_excel, # The function to process the uploaded file | |
| inputs=gr.File(type="filepath", label="Upload Excel File here. \t Be sure to check that the column headings in your upload are the same as in the Example files below. \t (Otherwise there will be Error during the processing)"), # File upload input | |
| examples=example_files, # Add the example files | |
| outputs=[ | |
| gr.File(label="Download the processed Excel File containing the ** Project Proposals ** for each Location~Problem paired combination"), # File download output | |
| gr.Textbox(label="Console Messages", lines=7, interactive=False) # Console messages output | |
| ], | |
| # title="Excel File Uploader", | |
| # title="Upload Excel file containing #TaxDirections → Download HyperLocal Project Proposals\n", | |
| title = ( | |
| "<p style='font-weight: bold; font-size: 25px; text-align: center;'>" | |
| "<span style='color: blue;'>Upload Excel file containing #TaxDirections</span> " | |
| # "<span style='color: brown; font-size: 35px;'>→ </span>" | |
| # "<span style='color: brown; font-size: 35px;'>⇒ ↣ ↠ </span>" | |
| "<span style='color: brown; font-size: 35px;'> " +a_random_object +" </span>" | |
| "<span style='color: green;'>Download HyperLocal Project Proposals</span>" | |
| "</p>\n" | |
| ), | |
| description=( | |
| "<p style='font-size: 12px; color: gray; text-align: center'>This tool allows for the systematic evaluation and proposal of solutions tailored to specific location-problem pairs, ensuring efficient resource allocation and project planning. For more information, visit <a href='https://santanban.github.io/TaxDirection/' target='_blank'>#TaxDirection weblink</a>.</p>" | |
| "<p style='font-weight: bold; font-size: 16px; color: blue;'>Upload an Excel file to process and download the result or use the Example files:</p>" | |
| "<p style='font-weight: bold; font-size: 15px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>" | |
| "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Processed output contains a Project Proposal for each Location~Problem paired combination (i.e. each cell).</p>" | |
| "<p style='font-weight: bold; font-size: 13px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>" | |
| "<p style='font-size: 12px; color: gray; text-align: center'>Note: The example files provided above are for demonstration purposes. Feel free to upload your own Excel files to see the results. If you have any questions, refer to the documentation-links or contact <a href='https://www.change.org/p/democracy-evolution-ensuring-humanity-s-eternal-existence-through-taxdirection' target='_blank'>support</a>.</p>" | |
| ) # Solid description with right-aligned second sentence | |
| ) | |
| # Launch the interface | |
| if __name__ == "__main__": | |
| interface.launch() |