Merge branch 'eason/main' into eason/fix_term_check

dict_util.py

@@ -0,0 +1,46 @@
+import csv
+import pickle
+
+### NOTICE: csv only accept two colomn input. but accept multi-time input.
+
+
+# 1_2_3, 1 is action, 2 is supply object, 3 is accept object
+def update_dict_csv(term_dict, f):
+    for rows in csv.reader(f):
+        if rows[0] in term_dict:
+            if rows[1] not in term_dict[rows[0]]:
+                term_dict[rows[0]] = term_dict[rows[0]]+[rows[1]]
+        else:
+            term_dict[rows[0]]=[rows[1]]
+    pass
+
+def export_dict_csv(term_dict, f):
+    for key, val in term_dict.items():
+        csv.writer(f).writerow([key, val])
+    pass
+
+def save_dict_pickle(term_dict, f):
+    pickle.dump(term_dict, f, pickle.HIGHEST_PROTOCOL)
+
+def update_csv_pickle(pickle_f, csv_f):
+    term_dict = pickle.load(pickle_f)
+    for rows in csv.reader(csv_f):
+        if rows[0] in term_dict:
+            if rows[1] not in term_dict[rows[0]]:
+                term_dict[rows[0]] = term_dict[rows[0]]+[rows[1]]
+        else:
+            term_dict[rows[0]]=[rows[1]]
+    #save to pickle file, highest protocal to get better performance
+    pickle.dump(term_dict, pickle_f, pickle.HIGHEST_PROTOCOL)
+
+
+#demo
+term_dict_sc2 = {}
+with open("./finetune_data/dict_enzh.csv", 'r', encoding='utf-8') as f:
+    update_dict_csv(term_dict_sc2,f)
+
+with open("../test.csv", "w", encoding='utf-8') as w:
+    export_dict_csv(term_dict_sc2,w)
+
+## for load pickle, just:
+# pickle.load(f)

finetune_data/dict_enzh.csv

@@ -1,4 +1,4 @@
-barracks,兵营
+barracks,兵营
 zerg,虫族
 protoss,神族
 terran,人族
@@ -7,6 +7,7 @@ forge,锻炉
 blink,闪现
 evolution chamber,进化腔
 cybernetics core,控制芯核
+cybercore,控制芯核
 enhanced shockwaves,EMP范围
 gravitic boosters,ob速度
 armory,军械库
@@ -36,6 +37,7 @@ planetary fortress,大地堡
 battle cruiser,大和
 weapon refit,大和炮
 brood lord,大龙
+broodling,巢虫
 greater spire,大龙塔
 anabolic synthesis,大牛速度
 cyclone,导弹车
@@ -80,7 +82,7 @@ spine crawler,管子
 marauder,光头
 ghost,鬼兵
 arm silo with nuke,核弹
-carrier,黄金舰队
+carrier,航母
 hellion,火车
 hellbat,火车侠
 ravager,火蟑螂
@@ -175,3 +177,6 @@ glial reconstitution,蟑螂速度
 concussive shells,震撼弹
 stalker,追猎
 disruptor,自爆球
+zerg,虫族
+protross,神族
+terran,人族

finetune_data/dict_freq.txt

@@ -59,6 +59,7 @@ supply depot
 overlord
 pneumatized carapace
 mutalisk
+broodling
 spire
 viper
 flyer attacks
@@ -174,4 +175,19 @@ disruptor
 zerg
 protross
 terran
-starcraft
+starcraft
+TvT
+Maxpax
+showtime
+PvP
+ZvZ
+TvZ
+TvP
+ZvP
+PvZ
+PvT
+ZvT
+Florencio
+cybercore
+nest
+follow-up