app.py

from Evaluator import evaluate_predictions_for_list
# Generate the combined metrics data structures
import json
import pandas as pd
import os

import matplotlib.pyplot as plt

import gradio as gr


def gradio_interface(ground_truth_json, prediction_jsons):
    # prediction_folders = [r'C:\Users\kosti\OneDrive - Image Access Corp\ml-rnd\Products\Qwen2vl\qwen2_vl2b.json']
    engine_eval_results = evaluate_predictions_for_list(prediction_jsons, ground_truth_json, save_metrics_in_folder=True)

    # --------------------------------------------------------------------------------------------------------------- #
    # Define the path to the parent folder containing the engine subfolders
    # engine_folder = r"C:\Users\kosti\OneDrive - Image Access Corp\ml-rnd\data\EnginesSroieMetrics"
    engine_folder = os.getcwd()

    # Data structures to hold combined data
    field_metrics_combined = []
    file_metrics_combined = []
    combined_json_data = {}

    # Loop through each subdirectory in the parent folder
    for subdir in os.listdir(engine_folder):
        # Check if the folder ends with "_eval_results" (indicating it's an engine's result folder)
        if subdir.endswith("_eval_results"):
            engine_name = subdir.replace("_eval_results", "")

            # Construct the expected file paths for the field and file metrics CSVs and the YnY_hat JSON file
            field_metrics_path = os.path.join(engine_folder, subdir, f"{engine_name}_field_metrics.csv")
            file_metrics_path = os.path.join(engine_folder, subdir, f"{engine_name}_file_metrics.csv")
            json_path = os.path.join(engine_folder, subdir, f"{engine_name}_YnY_hat.json")

            # Check if the CSV and JSON files exist
            if os.path.exists(field_metrics_path) and os.path.exists(file_metrics_path) and os.path.exists(json_path):
                # Load the CSV files into Pandas DataFrames
                field_metrics_df = pd.read_csv(field_metrics_path)
                file_metrics_df = pd.read_csv(file_metrics_path)

                # Add a column to indicate the engine name for comparison
                field_metrics_df["engine"] = engine_name
                file_metrics_df["engine"] = engine_name

                # Append to the combined lists
                field_metrics_combined.append(field_metrics_df)
                file_metrics_combined.append(file_metrics_df)

                # Load the JSON data and merge it into the combined_json_data dictionary
                with open(json_path, 'r') as f:
                    json_data = json.load(f)
                    combined_json_data[engine_name] = json_data
            else:
                print(f"Missing metrics or JSON files for engine: {engine_name}")

    # Concatenate all the data into a single DataFrame for each metric type
    field_metrics_combined = pd.concat(field_metrics_combined, ignore_index=True)
    file_metrics_combined = pd.concat(file_metrics_combined, ignore_index=True)

    # Pivot the data so that we have attributes as rows and engines/metrics|similarities as columns
    field_metrics_combined = field_metrics_combined.pivot_table(index='attribute', columns='engine', values=['accuracy', 'avg_similarity'])
    file_metrics_combined = file_metrics_combined.pivot_table(index='filename', columns='engine',
                                                              values=[col for col in file_metrics_combined.columns if col.endswith('_similarity')])

    # Flatten the multi-level columns for easier understanding
    field_metrics_combined.columns = [f'{metric}_{engine}' for metric, engine in field_metrics_combined.columns]
    file_metrics_combined.columns = [f'{metric}_{engine}' for metric, engine in file_metrics_combined.columns]

    # Save the combined JSON data to a file

    field_metrics_combined.to_csv(os.path.join(engine_folder, 'field_metrics_combined.csv'))
    file_metrics_combined.to_csv(os.path.join(engine_folder, 'file_metrics_combined.csv'))
    with open(os.path.join(engine_folder, 'combined_YnY_hat.json'), 'w') as outfile:
        json.dump(combined_json_data, outfile, indent=4)
    print(f"Combined field metrics, file metrics, and JSON data have been saved successfully in {engine_folder}")

    # --------------------------------------------------------------------------------------------------------------- #
    # Plot accuracy and similarity in two separate figures
    mean_accuracy_similarity_scores = field_metrics_combined.mean()
    # print(f'Mean accuracy and similarity scores for engines:\n {mean_accuracy_similarity_scores}')
    # Load the combined field and file metrics files
    # field_metrics_combined_path = r'C:\Users\kosti\OneDrive - Image Access Corp\ml-rnd\data\EnginesSroieMetrics\field_metrics_combined.csv'
    field_metrics_combined_path = 'field_metrics_combined.csv'

    field_metrics_combined = pd.read_csv(field_metrics_combined_path)

    # Separate the columns for accuracy and average similarity
    accuracy_columns = [col for col in field_metrics_combined.columns if col.startswith('accuracy')]
    similarity_columns = [col for col in field_metrics_combined.columns if col.startswith('avg_similarity')]

    # Accuracy plot
    fig1, ax1 = plt.subplots(figsize=(10, 6))
    field_metrics_combined.set_index('attribute')[accuracy_columns].plot(kind='bar', ax=ax1, width=0.8)
    ax1.set_title('Accuracy Comparison Across Engines')
    ax1.set_xlabel('Attributes')
    ax1.set_ylabel('Accuracy (%)')
    plt.xticks(rotation=45)
    ax1.legend(loc='lower right')
    plt.tight_layout()

    # Similarity plot
    fig2, ax2 = plt.subplots(figsize=(10, 6))
    field_metrics_combined.set_index('attribute')[similarity_columns].plot(kind='bar', ax=ax2, width=0.8)
    ax2.set_title('Average Similarity Comparison Across Engines')
    ax2.set_xlabel('Attributes')
    ax2.set_ylabel('Average Similarity (%)')
    plt.xticks(rotation=45)
    ax2.legend(loc='lower right')
    plt.tight_layout()

    # Cleanup by deleting the qwen2_vl2b_eval_results folder
    remove_eval_results_folder_after_plots = True
    if remove_eval_results_folder_after_plots:
        for subdir in os.listdir(engine_folder):
            if subdir.endswith("_eval_results"):
                import shutil
                shutil.rmtree(os.path.join(engine_folder, subdir))
        os.remove(os.path.join(engine_folder, 'combined_YnY_hat.json'))
        os.remove(os.path.join(engine_folder, 'field_metrics_combined.csv'))
        os.remove(os.path.join(engine_folder, 'file_metrics_combined.csv'))

    return fig1, fig2, mean_accuracy_similarity_scores


gradio_app = gr.Interface(
    fn=gradio_interface,
    inputs=[gr.File(label='Ground Truth', file_types=['json']),
            gr.File(label='Json Files with Predictions', file_count='multiple', file_types=['json'])],
    outputs=[gr.Plot(), gr.Plot(), gr.Text(label='Accuracy Comparison Across Engines')],
    examples=[['data/ground_truth/sroie_ground_truth.json',
               [
                   'data/Predictions/amazon.json',
                   'data/Predictions/google_expense.json',
                   'data/Predictions/microsoft.json',
                   'data/Predictions/llr.json',
                   'data/Predictions/qwen2_vl2b.json',
                   'data/Predictions/qwen2_vl7b.json',
                ]]]
)

gradio_app.launch(debug=True, share=True)