T2V-CompBench_Leaderboard / draw_sub_dimension.py
Kaiyue's picture
Update draw_sub_dimension.py
9f82aab verified
import pandas as pd
import numpy as np
import os
import csv
import math
def sub_consist_attr(model,high=15,low=1):
color_indices = [1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,57,58,59,63,65,68,70,71,73,75,76,77,78,79,80,81,83,98,99]
color_indices += list(range(101, 161))
shape_indices = [64,22,23,24,25,26,27,28,29,30,31,32,33,34,35,72,74,84,85,86,87,92,94,95,100] + [161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175] # 161-175
texture_indices = [36,37,38,39,40,41,42,43,44,45,46,47,48,60,61,62,69,82,88,89,90,91,93,96,97] + list(range(176, 191)) # 176-190
human_indices = [49,50,51,52,53,54,55,56,66,67] + list(range(191, 201)) # 191-200
interval = high - low
score = []
df = pd.read_csv(model)
total_videos = df.shape[0] - 1
color = []
for i in color_indices:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
color.append((s-low)/interval)
shape = []
for i in shape_indices:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
shape.append((s-low)/interval)
texture = []
for i in texture_indices:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
texture.append((s-low)/interval)
color_score = "{:.4f}".format(sum(color)/len(color))
shape_score = "{:.4f}".format(sum(shape)/len(shape))
texture_score = "{:.4f}".format(sum(texture)/len(texture))
print(model)
print(len(color),len(shape),len(texture))
print(color_score, shape_score, texture_score)
return color_score, shape_score, texture_score
def sub_action(model,high=10,low=1):
common_ind = list(range(1, 23)) + list(range(46, 101)) + list(range(101,184))
uncommon_ind = list(range(23, 46)) + list(range(184,201))
interval = high - low
score = []
df = pd.read_csv(model)
total_videos = df.shape[0] - 1
common = []
for i in common_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
common.append((s-low)/interval)
uncommon = []
for i in uncommon_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
uncommon.append((s-low)/interval)
common_score = "{:.4f}".format(sum(common)/len(common))
uncommon_score = "{:.4f}".format(sum(uncommon)/len(uncommon))
print(model)
print(len(common),len(uncommon))
print(common_score,uncommon_score)
return common_score,uncommon_score
def sub_interaction(model,high=10,low=1):
physical_ind = list(range(1, 50)) + list(range(101, 152))
social_ind = list(range(50, 101)) + list(range(152,201))
interval = high - low
score = []
df = pd.read_csv(model)
total_videos = df.shape[0] - 1
physical = []
for i in physical_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
physical.append((s-low)/interval)
social = []
for i in social_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
social.append((s-low)/interval)
physical_score = "{:.4f}".format(sum(physical)/len(physical))
social_score = "{:.4f}".format(sum(social)/len(social))
print(model)
print(len(physical),len(social))
print(physical_score, social_score)
return physical_score, social_score
def sub_spatial(model):
left_ind = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 88, 89, 90, 91, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 181, 182, 183, 184] # Example indices; replace with your actual indices
right_ind = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 92, 93, 94, 95, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 185, 186, 187, 188]
above_ind = [29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 189, 190, 191]
below_ind = [96, 97, 98, 99, 100, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 126, 192, 193, 194]
front_ind = [58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 195, 196, 197]
behind_ind = [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 198, 199, 200]
record = {}
df = pd.read_csv(model) # Replace with your CSV file path
total_videos = df.shape[0] - 1
scores = df.iloc[:, -1].tolist()
scores = scores[:200]
left = []
for i in left_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
left.append(s)
right = []
for i in right_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
right.append(s)
above = []
for i in above_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
above.append(s)
below = []
for i in below_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
below.append(s)
front = []
for i in front_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
front.append(s)
behind = []
for i in behind_ind:
for j in range(total_videos):
if df.iloc[j, 0][:4] == f"{i:04d}":
s = float(df.iloc[j, -1])
behind.append(s)
all_spatial = [left,right,above,below,front,behind]
for i,left in enumerate(all_spatial):
count_ge_0_4 = 0 # Count of scores >= 0.4
count_gt_0_4 = 0 # Count of scores > 0.4
count_eq_0_4 = 0 # Count of scores == 0.4
# Count the scores based on the conditions
scores_gt_0_4 = []
for s in left:
if round(s,4) >= 0.4:
count_ge_0_4 += 1
if round(s,4) > 0.4:
count_gt_0_4 += 1
scores_gt_0_4.append(s)
if round(s,4) == 0.4:
count_eq_0_4 += 1
ge_percent = count_ge_0_4 / len(left) #Comp
gt_percent = count_gt_0_4 / count_ge_0_4 #Acc.
gt_avg = sum(scores_gt_0_4) / len(scores_gt_0_4)#Acc. Score
eq_percent = count_eq_0_4 / count_ge_0_4
record[f"#{i+1}"] = [ge_percent,gt_percent,gt_avg,eq_percent,count_ge_0_4,count_gt_0_4,scores_gt_0_4,count_eq_0_4]
########################################################################################################################
left_count_ge_0_4 = record["#1"][4]
left_count_gt_0_4 = record["#1"][5]
left_scores_gt_0_4 = record["#1"][6]
right_count_ge_0_4 = record["#2"][4]
right_count_gt_0_4 = record["#2"][5]
right_scores_gt_0_4 = record["#2"][6]
left_right_ge_percent = (left_count_ge_0_4+right_count_ge_0_4) / (len(left)+len(right))
left_right_gt_percent = (left_count_gt_0_4+right_count_gt_0_4)/(left_count_ge_0_4+right_count_ge_0_4)
left_right_gt_avg = (sum(left_scores_gt_0_4)+sum(right_scores_gt_0_4)) / (left_count_gt_0_4+right_count_gt_0_4)
record["left_right"] = [round(left_right_ge_percent,4),round(left_right_gt_percent,4),round(left_right_gt_avg,4)]
########################################################################################################################
above_count_ge_0_4 = record["#3"][4]
above_count_gt_0_4 = record["#3"][5]
above_scores_gt_0_4 = record["#3"][6]
below_count_ge_0_4 = record["#4"][4]
below_count_gt_0_4 = record["#4"][5]
below_scores_gt_0_4 = record["#4"][6]
_2d_ge_percent = (left_count_ge_0_4 + right_count_ge_0_4 + above_count_ge_0_4 + below_count_ge_0_4) / (len(left)+len(right)+len(above)+len(below))
_2d_gt_percent = (left_count_gt_0_4 + right_count_gt_0_4 + above_count_gt_0_4 + below_count_gt_0_4)/(left_count_ge_0_4 + right_count_ge_0_4 + above_count_ge_0_4 + below_count_ge_0_4)
_2d_gt_avg = (sum(left_scores_gt_0_4) + sum(right_scores_gt_0_4) + sum(above_scores_gt_0_4) + sum(below_scores_gt_0_4)) / (left_count_gt_0_4 + right_count_gt_0_4 + above_count_gt_0_4 + below_count_gt_0_4)
record["2d"] = [f"{_2d_ge_percent:.0%}",f"{_2d_gt_percent:.0%}",round(_2d_gt_avg,4)]
########################################################################################################################
front_count_ge_0_4 = record["#5"][4]
front_count_gt_0_4 = record["#5"][5]
front_scores_gt_0_4 = record["#5"][6]
behind_count_ge_0_4 = record["#6"][4]
behind_count_gt_0_4 = record["#6"][5]
behind_scores_gt_0_4 = record["#6"][6]
_3d_ge_percent = (front_count_ge_0_4+behind_count_ge_0_4) / (len(front)+len(behind))
_3d_gt_percent = (front_count_gt_0_4+behind_count_gt_0_4)/(front_count_ge_0_4+behind_count_ge_0_4)
_3d_gt_avg = (sum(front_scores_gt_0_4)+sum(behind_scores_gt_0_4)) / (front_count_gt_0_4+behind_count_gt_0_4)
record["3d"] = [round(_3d_ge_percent,4),round(_3d_gt_percent,4),round(_3d_gt_avg,4)]
print(model)
print(len(left),len(right),len(above),len(below))
print(record["2d"])
coexist = record["2d"][0]
acc = record["2d"][1]
acc_score = record["2d"][2]
return coexist, acc, acc_score
def object_score(obj1_net_left,left_thresh,obj1_net_up,up_thresh,d_1):
correct_direction = False
W = 856
H = 480
score_tmp = 0
obj1_net_left = float(obj1_net_left)*100/W #normalize, map: 100x100
obj1_net_up = float(obj1_net_up)*100/H
net_distance = math.sqrt(obj1_net_left**2+obj1_net_up**2)
if d_1 == "left":
if obj1_net_left>left_thresh:
correct_direction = True
elif d_1 == "right":
if obj1_net_left<-left_thresh:
correct_direction = True
elif d_1 == "up":
if obj1_net_up>up_thresh:
correct_direction = True
elif d_1 == "down":
if obj1_net_up<-up_thresh:
correct_direction = True
else:
print("direction not in [left, right, up, down]")
return correct_direction,net_distance
def sub_motion(model):
#mid point y:240, x:428 height = 480, width = 856
left_thresh = 5 #5%
up_thresh = 5 #5%
distance = []
direction = []
with open(model, 'r') as file1:
reader1 = csv.reader(file1)
lines = list(reader1)
vid_num = (len(lines)-1)//2
for i in range(vid_num):
id = lines[i*2+1][0]
d_1 = lines[i*2+1][3]
d_2 = lines[i*2+2][5]
obj1 = lines[i*2+1][2]
obj2 = lines[i*2+2][4]
obj1_net_left = lines[i*2+1][6]
obj1_net_up = lines[i*2+1][7]
obj2_net_left = lines[i*2+2][6]
obj2_net_up = lines[i*2+2][7]
correct_direction = False
score_tmp = 0
if d_1!="" and d_2=="": #only 1 object
if obj1_net_left != "": #1 object detected Comp
correct_direction,net_distance = object_score(obj1_net_left,left_thresh,obj1_net_up,up_thresh,d_1)
distance.append(net_distance)
direction.append(correct_direction) # true false
motion_level = sum(distance)/len(distance)
acc = sum(direction)/len(direction)
print(model)
print(len(distance)) #< 165
print(len(direction))
print(round(motion_level,2), f"{acc:.0%}",)
return round(motion_level,2), f"{acc:.0%}"
def read_score(model):
with open(model, 'r') as file:
reader = csv.reader(file)
lines = list(reader)
if lines[-1][0]=="score: " or lines[-1][0]=="Score: ":
score = float(lines[-1][-1])
else:
return "No score found"
return round(score,4)