Upload 3 files

app.py

@@ -0,0 +1,441 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+import os
+import io
+import gradio as gr
+import pandas as pd
+import datetime
+import zipfile
+import numpy as np
+
+from constants import *
+from draw_sub_dimension import *
+from huggingface_hub import Repository
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+global data_component, filter_component
+
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str, # required
+    revision_name_textbox: str,
+    access_type: str,
+    model_link: str, # required
+    team_name: str,
+    contact_email: str, # required
+    model_publish: str,
+    model_resolution: str,
+    model_frame: str,
+    model_fps: str,
+    model_video_length: str,
+    model_checkpoint: str,
+    model_commit_id: str,
+    model_video_format: str
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    
+    if  model_link == '' or model_name_textbox == '' or contact_email == '':
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull() 
+    
+    
+    now = datetime.datetime.now()
+    upload_date = now.strftime("%Y-%m-%d")  # Capture update time
+    upload_time = now.strftime("%Y-%m-%d_%H-%M-%S")
+    filename = f"{model_name_textbox}_{upload_time}"
+    
+    with open(f'{SUBMISSION_NAME}/{filename}.zip','wb') as f:
+        f.write(input_file)
+  
+  
+    csv_data = pd.read_csv(CSV_PATH)
+
+    if revision_name_textbox == '':
+        col = csv_data.shape[0]
+        model_name = model_name_textbox.replace(',',' ')
+    else:
+        model_name = revision_name_textbox.replace(',',' ')
+        model_name_list = csv_data['Model Name (clickable)']
+        name_list = [name.split(']')[0][1:] for name in model_name_list]
+        if revision_name_textbox not in name_list:
+            col = csv_data.shape[0]
+        else:
+            col = name_list.index(revision_name_textbox)    
+
+    model_name = '[' + model_name + '](' + model_link + ')'
+
+    folder = f'{SUBMISSION_NAME}/{filename}'
+    os.makedirs(folder, exist_ok=True)
+    with zipfile.ZipFile(io.BytesIO(input_file), 'r') as zip_ref:
+        zip_ref.extractall(folder)
+
+    required_files = [
+        "_consistent_attr_score.csv",
+        "_dynamic_attr_score.csv",
+        "_spatial_score.csv",
+        "_motion_score.csv",
+        "_motion_back_fore.csv",
+        "_action_binding_score.csv",
+        "_object_interactions_score.csv",
+        "_numeracy_video.csv",
+    ]
+    
+    score_1 = score_2 = score_3 = score_4 = score_5 = score_6 = score_7 = "N/A"
+    color_score = shape_score = texture_score = coexist = acc = acc_score = "N/A"
+    motion_level = motion_acc = common_score = uncommon_score = physical_score = social_score = "N/A"
+    
+    for i,suffix in enumerate(required_files):
+        for sub_folder in os.listdir(folder):
+            if sub_folder.startswith('.') or sub_folder.startswith('__'):
+                print(f"Skip the file: {sub_folder}")
+                continue
+            
+            cur_sub_folder = os.path.join(folder, sub_folder) #user_upload_zip_name
+            if os.path.isdir(cur_sub_folder):
+                for file in os.listdir(cur_sub_folder):
+                    if file.endswith(suffix):
+                        print("FILE exist",file)
+                        filepath = os.path.join(cur_sub_folder,file)
+                        if i==0: 
+                            score_1 = read_score(filepath)
+                            color_score, shape_score, texture_score = sub_consist_attr(filepath)
+                        elif i==1:
+                            score_2 = read_score(filepath)
+                        elif i==2:
+                            score_3 = read_score(filepath)
+                            coexist, acc, acc_score = sub_spatial(filepath)
+                        elif i==3:
+                            score_4 = read_score(filepath)
+                        elif i==4:
+                            motion_level, motion_acc = sub_motion(filepath)
+                        elif i==5:
+                            score_5 = read_score(filepath)
+                            common_score,uncommon_score = sub_action(filepath)
+                        elif i==6:
+                            score_6 = read_score(filepath)
+                            physical_score, social_score = sub_interaction(filepath)
+                        elif i==7:
+                            score_7 = read_score(filepath)
+                    
+    # add new data  
+    if team_name =='' or 'compbench' in team_name.lower():
+        evaluate_team = ("User Upload")
+    else:
+        evaluate_team = team_name
+     
+    new_data = [model_name,evaluate_team,upload_date,score_1,score_2,score_3,score_4,score_5,score_6,score_7,color_score, shape_score, texture_score,coexist, acc, acc_score,motion_level, motion_acc,common_score,uncommon_score,physical_score, social_score]
+    print(new_data)
+
+    csv_data.loc[col] = new_data
+    csv_data = csv_data.to_csv(CSV_PATH, index=False)
+    
+    new_info = [model_name,upload_time,team_name,model_publish,model_resolution,model_frame,model_fps,model_video_length,model_checkpoint,model_commit_id,model_video_format,access_type,contact_email,model_link]
+    with open(INFO_PATH, mode='a', newline='') as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(new_info)  
+
+    submission_repo.push_to_hub()
+
+    print("success update", model_name)
+    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+
+
+def calculate_selected_score(df, selected_columns):
+    selected_task = [i for i in selected_columns if i in TASK_INFO]
+    
+    selected_task_score = df[selected_task].mean(axis=1, skipna=True)
+    if selected_task_score.isna().any().any():
+        return selected_task_score.fillna(0.0)
+    return selected_task_score.fillna(0.0)
+
+def get_final_score(df, selected_columns):
+    df[TASK_INFO] = df[TASK_INFO].replace("N/A", np.nan)
+    df[TASK_INFO] = df[TASK_INFO].apply(pd.to_numeric, errors='coerce')
+    final_score = df[TASK_INFO].mean(axis=1, skipna=True)
+    final_score = round(final_score,4)
+    
+    if 'Total Avg. Score' in df:
+        df['Total Avg. Score'] = final_score
+    else:
+        df.insert(1, 'Total Avg. Score', final_score)
+         
+    selected_score = calculate_selected_score(df, selected_columns)
+    selected_score = round(selected_score,4)
+    
+    if 'Selected Avg. Score' in df:
+        df['Selected Avg. Score'] = selected_score
+    else:
+        df.insert(1, 'Selected Avg. Score', selected_score)
+    return df
+
+
+def get_baseline_df():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()    
+    df = pd.read_csv(CSV_PATH)
+    df = get_final_score(df, checkbox_group.value)
+    df = df.sort_values(by="Selected Avg. Score", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    df = df[df['Evaluated by'] == 'T2V-CompBench Team'] 
+    return df
+
+def get_baseline_df_sub():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()    
+    df = pd.read_csv(CSV_PATH)
+    df = get_final_score(df, checkbox_group.value)
+    df = df.sort_values(by="Selected Avg. Score", ascending=False)
+    
+    present_columns = MODEL_INFO[:-2] + SUB_TASK_INFO + MODEL_INFO[-2:] 
+    print(present_columns)
+    
+    df = df[present_columns]
+    df = df[df['Evaluated by'] == 'T2V-CompBench Team'] 
+    return df
+
+
+
+def get_all_df(selected_columns, csv=CSV_PATH): 
+    df = pd.read_csv(csv)
+    df = get_final_score(df, selected_columns)
+    df = df.sort_values(by="Selected Avg. Score", ascending=False)
+    return df
+
+
+# select function
+def category_checkbox_change(selected_columns, only_compbench_team):
+
+    updated_data = get_all_df(selected_columns, CSV_PATH)
+    if only_compbench_team:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'T2V-CompBench Team'] 
+
+    # columns:
+    selected_columns = [item for item in TASK_INFO if item in selected_columns]
+    present_columns = MODEL_INFO + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Avg. Score", ascending=False)
+
+    updated_headers = present_columns
+    update_datatype = [DATA_TITLE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component
+
+def category_checkbox_change_sub(selected_columns, selected_columns_sub,only_compbench_team):
+    updated_data = get_all_df(selected_columns, CSV_PATH)
+    if only_compbench_team:
+        updated_data = updated_data[updated_data['Evaluated by'] == 'T2V-CompBench Team'] 
+
+    # columns:
+    selected_columns = [item for item in SUB_TASK_INFO if item in selected_columns_sub]
+    present_columns = MODEL_INFO[:-2] + selected_columns + MODEL_INFO[-2:]
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Avg. Score", ascending=False)
+
+    updated_headers = present_columns
+    update_datatype = [SUB_DATA_TITLE_TYPE[SUB_COLUMN_NAMES.index(x)] for x in updated_headers]
+
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component
+
+
+block = gr.Blocks()
+
+with block:
+    gr.Markdown(
+        LEADERBOARD_INTRODUCTION 
+    )
+    gr.HTML(
+        LEADERBOARD_INTRODUCTION_HTML 
+    )
+    gr.Markdown(
+        LEADERBOARD_INTRODUCTION_2
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # Table 1
+        with gr.TabItem("📊 T2V-CompBench", elem_id="compbench-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=14,
+                    )
+    
+     
+            with gr.Row():
+                compbench_team_filter = gr.Checkbox(
+                    label="Evaluated by T2V-CompBench Team (Uncheck to view all submissions)",
+                    value=True,
+                    interactive=True
+                )
+
+            with gr.Row():
+                # selection for column part:
+                checkbox_group = gr.CheckboxGroup(
+                    choices=TASK_INFO,
+                    value=TASK_INFO,
+                    label="Evaluation Category",
+                    interactive=True,
+                )
+            
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            checkbox_group.change(fn=category_checkbox_change, inputs=[checkbox_group, compbench_team_filter], outputs=data_component)
+            compbench_team_filter.change(fn=category_checkbox_change, inputs=[checkbox_group, compbench_team_filter], outputs=data_component)
+        
+        # Table 2
+        with gr.TabItem("🗂️ Sub-Dimension", elem_id="compbench-tab-table", id=2):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=14,
+                    )
+            with gr.Row():
+                compbench_team_filter_sub = gr.Checkbox(
+                    label="Evaluated by T2V-CompBench Team (Uncheck to view all submissions)",
+                    value=True,
+                    interactive=True
+                    )
+            with gr.Row():
+                # selection for column part:
+                checkbox_group_sub = gr.CheckboxGroup(
+                    choices=SUB_TASK_INFO,
+                    value=SUB_TASK_INFO,
+                    label="Evaluation Sub-Dimensions",
+                    interactive=True,
+                )
+            
+            data_component_sub = gr.components.Dataframe(
+                value=get_baseline_df_sub, 
+                headers=SUB_COLUMN_NAMES,
+                type="pandas", 
+                datatype=SUB_DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            checkbox_group_sub.change(fn=category_checkbox_change_sub, inputs=[checkbox_group,checkbox_group_sub, compbench_team_filter_sub], outputs=data_component_sub)
+            compbench_team_filter_sub.change(fn=category_checkbox_change_sub, inputs=[checkbox_group,checkbox_group_sub, compbench_team_filter_sub], outputs=data_component_sub)
+        
+        
+        
+        # Table 3
+        with gr.TabItem("📝 About", elem_id="compbench-tab-table", id=3):
+            gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
+        
+        # Table 4: table submission 
+        with gr.TabItem("🚀 Submit here! ", elem_id="compbench-tab-table", id=4):
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation CSV files here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("Here is a required field", elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model Name", placeholder="Required field"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name(Optional)", placeholder="If you need to update the previous results, please fill in this line"
+                    )
+                    access_type = gr.Dropdown(choices=["Open Source", "Ready to Open Source", "API", "Close"], value=None,label="Please select the way user can access your model. You can update the content by revision_name, or contact the T2V-CompBench Team.")
+
+
+                with gr.Column():
+                    model_link = gr.Textbox(
+                        label="Project Page/Paper Link/Github/HuggingFace Repo", placeholder="Required field. If filling in the wrong information, your results may be removed."
+                    )
+                    team_name = gr.Textbox(
+                        label="Your Team Name(If left blank, it will be user upload)", placeholder="User Upload"
+                    )
+                    contact_email = gr.Textbox(
+                        label="E-Mail(Will not be displayed)", placeholder="Required field"
+                    )
+            with gr.Row():
+                gr.Markdown("The following is optional and will be synced to [GitHub] (https://t2v-compbench.github.io/)", elem_classes="markdown-text")
+            with gr.Row():
+                    model_publish = gr.Textbox(label="Time of Publish", placeholder="1970-01-01")
+                    model_resolution = gr.Textbox(label="Resolution", placeholder="width x height")
+                    model_frame = gr.Textbox(label="Frame Count", placeholder="int")
+                    model_fps = gr.Textbox(label="FPS", placeholder="int")
+                    model_video_length = gr.Textbox(label="Video Duration(s)", placeholder="float(2.0)")
+                    model_checkpoint = gr.Textbox(label="Model Checkpoint", placeholder="optional")
+                    model_commit_id = gr.Textbox(label="Github commit id", placeholder='optional')
+                    model_video_format = gr.Textbox(label="Video Format", placeholder='mp4/gif')
+            with gr.Column():
+                input_file = gr.components.File(label = "Click to Upload a ZIP File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval!")
+                submit_succ_button = gr.Markdown("Submit Success! Please press refresh and return to LeaderBoard!", visible=False)
+                fail_textbox = gr.Markdown(':bangbang:Please ensure that the `Model Name`, `Project Page`, and `Email` are filled in correctly.',visible=False)
+                
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        access_type,
+                        model_link,
+                        team_name,
+                        contact_email,
+                        
+                        model_publish,
+                        model_resolution,
+                        model_frame,
+                        model_fps,
+                        model_video_length,
+                        model_checkpoint,
+                        model_commit_id,
+                        model_video_format
+                    ],
+                    outputs=[submit_button, submit_succ_button, fail_textbox]
+                )
+                
+        
+    def refresh_data():
+        value1 = get_baseline_df()
+        return value1
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(category_checkbox_change, inputs=[checkbox_group, compbench_team_filter], outputs=data_component)
+      
+
+block.launch()

constants.py

@@ -0,0 +1,100 @@
+import os
+# this is .py for store constants 
+MODEL_INFO = [
+    "Model Name (clickable)",
+    "Evaluated by",
+    "Date",
+    "Total Avg. Score",
+    "Selected Avg. Score",
+    ]
+
+TASK_INFO = [
+    "Consistent Attribute Binding",
+    "Dynamic Attribute Binding",
+    "Spatial Relationships",
+    "Motion Binding",
+    "Action Binding",
+    "Object Interactions",
+    "Generative Numeracy",
+    ]
+
+SUB_TASK_INFO = [
+    "Consistent Attribute Binding-Color",
+    "Consistent Attribute Binding-Shape",
+    "Consistent Attribute Binding-Texture",
+    "2D Spatial Relationships-Coexist",
+    "2D Spatial Relationships-Acc.",
+    "2D Spatial Relationships-Acc.Score",
+    "Motion Binding-Motion Level",
+    "Motion Binding-Acc.",
+    "Action Binding-Common",
+    "Action Binding-Uncommon",
+    "Object Interactions-Physical",
+    "Object Interactions-Social",
+]
+
+
+
+SUBMISSION_NAME = "T2V-CompBench_leaderboard_submission"
+SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Kaiyue/", SUBMISSION_NAME)
+CSV_PATH = "./T2V-CompBench_leaderboard_submission/results.csv"
+INFO_PATH = "./T2V-CompBench_leaderboard_submission/model_info.csv"
+
+
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+DATA_TITLE_TYPE = ['markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number','number']
+
+SUB_COLUMN_NAMES = MODEL_INFO + SUB_TASK_INFO
+SUB_DATA_TITLE_TYPE = ['markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number','number', 'number','number', 'number','number', 'number']
+
+
+LEADERBOARD_INTRODUCTION  = """# T2V-CompBench Leaderboard
+     
+    🏆 Welcome to the leaderboard of the **T2V-CompBench**! 🎦 *A Comprehensive Benchmark for Compositional Text-to-video Generation*  """
+
+LEADERBOARD_INTRODUCTION_HTML = """
+    <div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
+    <a href='https://github.com/KaiyueSun98/T2V-CompBench'><img src='https://img.shields.io/github/stars/KaiyueSun98/T2V-CompBench.svg?style=social&label=Official'></a>
+    <a href='https://arxiv.org/abs/2407.14505'><img src='https://img.shields.io/badge/cs.CV-Paper-b31b1b?logo=arxiv&logoColor=red'></a>
+    <a href="https://t2v-compbench.github.io/"><img src="https://img.shields.io/badge/Project-Page-Green"></a>
+    </div>
+"""
+LEADERBOARD_INTRODUCTION_2  = """
+    - **1400 Prompts**: We analyze *1.67 million* real-user prompts to extract high-frequency nouns, verbs, and adjectives, resulting in a suite of 1,400 prompts.
+    - **7 Compositional Categories:** We evaluate multiple-object compositionality on attributes, actions, interactions, quantities, and spatio-temporal dynamics, covering 7 categories.
+    - **Evaluation metrics**: We design MLLM-based, Detection-based, and Tracking-based evaluation metrics for compositional T2V generation, all validated by human evaluations. 
+    - **Valuable Insights:** We provide insightful analysis on current models' ability, highlighting the significant challenge of compositional T2V generation.  
+    
+    **Join Leaderboard**: Please see the [instructions](https://t2v-compbench.github.io/) for 3 options to participate. One option is to follow [T2V-CompBench Usage info](https://t2v-compbench.github.io/), and upload the generated `.csv` files here. After clicking the `Submit Eval!` button, click the `Refresh` button.
+
+    **Model Information**: What are the details of these Video Generation Models? See [HERE](https://t2v-compbench.github.io/)
+    """
+    
+SUBMIT_INTRODUCTION = """# Submit on T2V-CompBench Introduction
+## 📮
+1. Please note that you need to obtain a list of `.csv` files by running T2V-CompBench in Github. You may conduct an [Offline Check](https://t2v-compbench.github.io/) before uploading.
+2. Then, pack these CSV files into a `ZIP` archive, ensuring that the top-level directory of the ZIP contains the individual CSV files. 
+3. Finally, upload the ZIP archive below.
+
+⚠️ Uploading generated videos of the model is invalid!
+
+⚠️ Submissions that do not correctly fill in the model name and model link may be deleted by the T2V-CompBench team. The contact information you filled in will not be made public. 
+"""
+
+LEADERBOARD_INFO = """
+- T2V-CompBench, a comprehensive benchmark for compositional text-to-video generation, consists of seven categories: **consistent attribute binding, dynamic attribute binding, spatial relationships, motion binding, action binding, object interactions, and generative numeracy**. 
+- For each category, we carefully design 200 prompts, resulting in **1400** in total, and sample generated videos from a set of T2V models. 
+- We propose three types of evaluation metrics: **MLLM-based, Detection-based, and Tracking-based metrics**, all specifically designed for compositional T2V generation and validated by human evaluations. 
+- We benchmark various T2V models, reveal their strengths and weaknesses by examining the results across **7 categories and 12 sub-dimensions**, meanwhile provide insightful analysis on compositional T2V generation.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{sun2024t2v,
+  title={T2v-compbench: A comprehensive benchmark for compositional text-to-video generation},
+  author={Sun, Kaiyue and Huang, Kaiyi and Liu, Xian and Wu, Yue and Xu, Zihan and Li, Zhenguo and Liu, Xihui},
+  journal={arXiv preprint arXiv:2407.14505},
+  year={2024}
+}"""
+
+

draw_sub_dimension.py

@@ -0,0 +1,359 @@
+import pandas as pd
+import numpy as np
+import os
+import csv
+import math
+
+def sub_consist_attr(model,high=15,low=1):
+    
+    color_indices = [1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,57,58,59,63,65,68,70,71,73,75,76,77,78,79,80,81,83,98,99]  
+    color_indices += list(range(101, 161)) 
+    shape_indices = [64,22,23,24,25,26,27,28,29,30,31,32,33,34,35,72,74,84,85,86,87,92,94,95,100] + [161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175]  # 161-175
+    texture_indices = [36,37,38,39,40,41,42,43,44,45,46,47,48,60,61,62,69,82,88,89,90,91,93,96,97] + list(range(176, 191))  # 176-190
+    human_indices = [49,50,51,52,53,54,55,56,66,67] + list(range(191, 201))  # 191-200
+    
+
+    interval = high - low
+    score = []
+    df = pd.read_csv(model)  
+    
+    color = []
+    for i in color_indices:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                color.append((s-low)/interval)
+                
+    shape = []
+    for i in shape_indices:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                shape.append((s-low)/interval)
+    texture = []
+    for i in texture_indices:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                texture.append((s-low)/interval)
+                
+    
+    color_score = "{:.4f}".format(sum(color)/len(color))
+    shape_score = "{:.4f}".format(sum(shape)/len(shape))
+    texture_score = "{:.4f}".format(sum(texture)/len(texture))
+    
+    print(model)
+    print(len(color),len(shape),len(texture))
+    print(color_score, shape_score, texture_score)
+
+    return color_score, shape_score, texture_score
+
+
+
+
+def sub_action(model,high=10,low=1):
+    
+    common_ind = list(range(1, 23)) + list(range(46, 101)) + list(range(101,184)) 
+    uncommon_ind = list(range(23, 46)) + list(range(184,201))
+    
+
+    interval = high - low
+    score = []
+    df = pd.read_csv(model)  
+    
+    common = []
+    for i in common_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                common.append((s-low)/interval)
+                
+    uncommon = []
+    for i in uncommon_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                uncommon.append((s-low)/interval)
+   
+    
+    common_score = "{:.4f}".format(sum(common)/len(common))
+    uncommon_score = "{:.4f}".format(sum(uncommon)/len(uncommon))
+   
+    print(model)
+    print(len(common),len(uncommon))
+    print(common_score,uncommon_score)
+
+    return common_score,uncommon_score
+
+
+def sub_interaction(model,high=10,low=1):
+    
+    physical_ind = list(range(1, 50)) + list(range(101, 152)) 
+    social_ind = list(range(50, 101)) + list(range(152,201))
+
+    interval = high - low
+    score = []
+    df = pd.read_csv(model)  
+    
+    physical = []
+    for i in physical_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                physical.append((s-low)/interval)
+                
+    social = []
+    for i in social_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                social.append((s-low)/interval)
+    
+                
+    
+    physical_score = "{:.4f}".format(sum(physical)/len(physical))
+    social_score = "{:.4f}".format(sum(social)/len(social))
+    
+    
+    print(model)
+    print(len(physical),len(social))
+    print(physical_score, social_score)
+
+    return physical_score, social_score
+
+
+
+def sub_spatial(model):
+ 
+    
+    left_ind = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 88, 89, 90, 91, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 181, 182, 183, 184]  # Example indices; replace with your actual indices
+    right_ind = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 92, 93, 94, 95, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 185, 186, 187, 188]
+    above_ind = [29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 189, 190, 191]
+    below_ind = [96, 97, 98, 99, 100, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 126, 192, 193, 194]
+    front_ind = [58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 195, 196, 197]
+    behind_ind = [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 198, 199, 200]
+
+
+    record = {}
+    df = pd.read_csv(model)  # Replace with your CSV file path
+
+    scores = df.iloc[:, -1].tolist()
+    scores = scores[:200]
+    
+    left = []
+    for i in left_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                left.append(s)
+    right = []
+    for i in right_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                right.append(s)
+                
+    above = []
+    for i in above_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                above.append(s)
+    below = []
+    for i in below_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                below.append(s)
+    
+    front = []
+    for i in front_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                front.append(s)
+    behind = []
+    for i in behind_ind:
+        for j in range(200):
+            if df.iloc[j, 0][:4] == f"{i:04d}":
+                s = float(df.iloc[j, -1])
+                behind.append(s)            
+
+    
+    all_spatial = [left,right,above,below,front,behind]
+    
+    for i,left in enumerate(all_spatial):
+        count_ge_0_4 = 0  # Count of scores >= 0.4
+        count_gt_0_4 = 0  # Count of scores > 0.4
+        count_eq_0_4 = 0  # Count of scores == 0.4
+
+        # Count the scores based on the conditions
+        scores_gt_0_4 = []
+        for s in left:
+            if round(s,4) >= 0.4:
+                count_ge_0_4 += 1
+            if round(s,4) > 0.4:
+                count_gt_0_4 += 1
+                scores_gt_0_4.append(s)
+            if round(s,4) == 0.4:
+                count_eq_0_4 += 1
+
+        ge_percent = count_ge_0_4 / len(left) #Comp
+        gt_percent = count_gt_0_4 / count_ge_0_4 #Acc.
+        gt_avg = sum(scores_gt_0_4) / len(scores_gt_0_4)#Acc. Score
+        eq_percent = count_eq_0_4 / count_ge_0_4
+        record[f"#{i+1}"] = [ge_percent,gt_percent,gt_avg,eq_percent,count_ge_0_4,count_gt_0_4,scores_gt_0_4,count_eq_0_4]
+       
+    ########################################################################################################################
+    left_count_ge_0_4 = record["#1"][4]
+    left_count_gt_0_4 = record["#1"][5]
+    left_scores_gt_0_4 = record["#1"][6]
+    
+    right_count_ge_0_4 = record["#2"][4]
+    right_count_gt_0_4 = record["#2"][5]
+    right_scores_gt_0_4 = record["#2"][6]
+
+    left_right_ge_percent = (left_count_ge_0_4+right_count_ge_0_4) / (len(left)+len(right))
+    left_right_gt_percent = (left_count_gt_0_4+right_count_gt_0_4)/(left_count_ge_0_4+right_count_ge_0_4)
+    left_right_gt_avg = (sum(left_scores_gt_0_4)+sum(right_scores_gt_0_4)) / (left_count_gt_0_4+right_count_gt_0_4)
+    
+    record["left_right"] = [round(left_right_ge_percent,4),round(left_right_gt_percent,4),round(left_right_gt_avg,4)]
+
+    ########################################################################################################################
+    
+    above_count_ge_0_4 = record["#3"][4]
+    above_count_gt_0_4 = record["#3"][5]
+    above_scores_gt_0_4 = record["#3"][6]
+    
+    below_count_ge_0_4 = record["#4"][4]
+    below_count_gt_0_4 = record["#4"][5]
+    below_scores_gt_0_4 = record["#4"][6]
+    
+    _2d_ge_percent = (left_count_ge_0_4 + right_count_ge_0_4 + above_count_ge_0_4 + below_count_ge_0_4) / (len(left)+len(right)+len(above)+len(below))
+    _2d_gt_percent = (left_count_gt_0_4 + right_count_gt_0_4 + above_count_gt_0_4 + below_count_gt_0_4)/(left_count_ge_0_4 + right_count_ge_0_4 + above_count_ge_0_4 + below_count_ge_0_4)
+    _2d_gt_avg = (sum(left_scores_gt_0_4) + sum(right_scores_gt_0_4) + sum(above_scores_gt_0_4) + sum(below_scores_gt_0_4)) / (left_count_gt_0_4 + right_count_gt_0_4 + above_count_gt_0_4 + below_count_gt_0_4)
+    
+    record["2d"] = [f"{_2d_ge_percent:.0%}",f"{_2d_gt_percent:.0%}",round(_2d_gt_avg,4)]
+    
+    ########################################################################################################################
+    front_count_ge_0_4 = record["#5"][4]
+    front_count_gt_0_4 = record["#5"][5]
+    front_scores_gt_0_4 = record["#5"][6]
+    
+    behind_count_ge_0_4 = record["#6"][4]
+    behind_count_gt_0_4 = record["#6"][5]
+    behind_scores_gt_0_4 = record["#6"][6]
+    
+
+    _3d_ge_percent = (front_count_ge_0_4+behind_count_ge_0_4) / (len(front)+len(behind))
+    _3d_gt_percent = (front_count_gt_0_4+behind_count_gt_0_4)/(front_count_ge_0_4+behind_count_ge_0_4)
+    _3d_gt_avg = (sum(front_scores_gt_0_4)+sum(behind_scores_gt_0_4)) / (front_count_gt_0_4+behind_count_gt_0_4)
+    
+    
+    record["3d"] = [round(_3d_ge_percent,4),round(_3d_gt_percent,4),round(_3d_gt_avg,4)]  
+    
+    
+    print(model)
+    print(len(left),len(right),len(above),len(below))
+    print(record["2d"])
+    coexist = record["2d"][0]
+    acc = record["2d"][1]
+    acc_score = record["2d"][2]
+    
+    return coexist, acc, acc_score
+    
+    
+    
+    
+def object_score(obj1_net_left,left_thresh,obj1_net_up,up_thresh,d_1):
+
+    correct_direction = False
+    
+    W = 856
+    H = 480
+    
+    score_tmp = 0
+    obj1_net_left = float(obj1_net_left)*100/W #normalize, map: 100x100
+    obj1_net_up = float(obj1_net_up)*100/H
+    net_distance = math.sqrt(obj1_net_left**2+obj1_net_up**2)
+    
+    if d_1 == "left":
+        if obj1_net_left>left_thresh:
+            correct_direction = True
+            
+    elif d_1 == "right":
+        if obj1_net_left<-left_thresh:
+            correct_direction = True
+    
+    elif d_1 == "up":
+        if obj1_net_up>up_thresh:
+            correct_direction = True
+     
+    elif d_1 == "down":
+        if obj1_net_up<-up_thresh:
+            correct_direction = True
+   
+    else:
+        print("direction not in [left, right, up, down]")
+        
+    return correct_direction,net_distance
+    
+    
+    
+def sub_motion(model): 
+
+    #mid point y:240, x:428  height = 480, width = 856
+    left_thresh = 5 #5%
+    up_thresh = 5 #5%
+
+   
+   
+
+    distance = []
+    direction = []
+
+    with open(model, 'r') as file1:
+        reader1 = csv.reader(file1)
+        lines = list(reader1)
+        vid_num = (len(lines)-1)//2
+        for i in range(vid_num):
+            id = lines[i*2+1][0]
+            d_1 = lines[i*2+1][3]
+            d_2 = lines[i*2+2][5]
+            obj1 = lines[i*2+1][2]
+            obj2 = lines[i*2+2][4]
+            obj1_net_left = lines[i*2+1][6]
+            obj1_net_up = lines[i*2+1][7]
+            obj2_net_left = lines[i*2+2][6]
+            obj2_net_up = lines[i*2+2][7]
+            correct_direction = False
+            score_tmp = 0
+            
+            if d_1!="" and d_2=="": #only 1 object
+                if obj1_net_left != "": #1 object detected  Comp
+                    correct_direction,net_distance = object_score(obj1_net_left,left_thresh,obj1_net_up,up_thresh,d_1)
+                    distance.append(net_distance)
+                    direction.append(correct_direction) # true false
+        
+        motion_level = sum(distance)/len(distance)
+        acc = sum(direction)/len(direction)   
+        
+        print(model)
+        print(len(distance)) #< 165
+        print(len(direction))   
+        print(round(motion_level,2), f"{acc:.0%}",)  
+        
+
+    return round(motion_level,2), f"{acc:.0%}"
+    
+
+def read_score(model):
+    with open(model, 'r') as file:
+        reader = csv.reader(file)
+        lines = list(reader)
+        
+        if lines[-1][0]=="score: " or lines[-1][0]=="Score: ":
+            score = float(lines[-1][-1])
+        else:
+            return "No score found"
+    return round(score,4)