Spaces:
Running
Running
| import os | |
| import torch | |
| import random | |
| import shutil | |
| import librosa | |
| import warnings | |
| import numpy as np | |
| import gradio as gr | |
| import librosa.display | |
| import matplotlib.pyplot as plt | |
| from utils import get_modelist, find_files, embed_img, TEMP_DIR | |
| from collections import Counter | |
| from model import EvalNet | |
| TRANSLATE = { | |
| "C0090": ["大笒", "da4_cen2"], | |
| "C0091": ["高音横笛", "Treble_heng2_di2"], | |
| "C0092": ["低音横笛", "Bass_heng2_di2"], | |
| "C0093": ["中音横笛", "Alto_heng2_di2"], | |
| "C0094": ["唢呐", "suo3_na"], | |
| "C0095": ["长唢呐", "chang2_suo3_na"], | |
| "C0096": ["小筚篥", "Treble_bi4_li"], | |
| "C0097": ["中音筚篥", "Alto_bi4_li"], | |
| "C0098": ["低音筚篥", "Bass_bi4_li"], | |
| "C0099": ["短箫", "duan3_xiao1"], | |
| "C0100": ["短箫(传统)", "duan3_xiao1_(traditional)"], | |
| "C0101": ["洞箫", "dong4_xiao1"], | |
| "C0113": ["尖子号", "jian1_zi3_hao4"], | |
| "C0114": ["尖子号2", "jian1_zi3_hao4_2"], | |
| "C0117": ["南音洞箫", "nan2_yin1_dong4_xiao1"], | |
| "C0123": ["南嗳仔", "nan2_ai1_zai3"], | |
| "C0124": ["大吹", "da4_chui1"], | |
| "C0182": ["长号", "chang2_hao4"], | |
| "C0183": ["老长号", "lao3_chang2_hao4"], | |
| "C0187": ["高音唢呐", "Treble_suo3_na"], | |
| "C0188": ["低音唢呐", "Bass_suo3_na"], | |
| "C0200": ["大芦笙", "da4_lu2_sheng1"], | |
| "C0201": ["小芦笙", "xiao3_lu2_sheng1"], | |
| "C0237": ["G调梆笛", "bang1_di2_in_G"], | |
| "C0243": ["高音键笙", "Treble_jian4_sheng1"], | |
| "C0244": ["传统笙", "traditional_sheng1"], | |
| "C0257": ["低音加键唢呐", "Bass_jia1_jian4_suo3_na"], | |
| "C0259": ["中音加键唢呐", "Alto_jia1_jian4_suo3_na"], | |
| "C0263": ["中音笙", "Alto_sheng1"], | |
| "C0264": ["低音笙", "Bass_sheng1"], | |
| "C0265": ["管子", "guan3_zi"], | |
| "C0280": ["A调曲笛", "qu3_di2_in_A"], | |
| "C0281": ["G调新笛", "xin1_di2_in_G"], | |
| "C0282": ["萧", "xiao1"], | |
| "C0283": ["埙", "xun1"], | |
| "C0296": ["唢呐2", "suo3_na_2"], | |
| "C0303": ["小闷笛", "xiao3_men1_di2"], | |
| "C0304": ["侗笛", "dong4_di2"], | |
| "C0305": ["德", "de2"], | |
| "C0306": ["拉祜族葫芦笙", "la1_hu2_zu2_hu2_lu2_sheng1"], | |
| "C0308": ["吐良", "tu3_liang2"], | |
| "C0309": ["葫芦丝", "hu2_lu2_si1"], | |
| "C0310": ["F调巴乌", "ba1_wu1_in_F"], | |
| "C0311": ["俄比", "e2_bi3"], | |
| "C0316": ["侗巴", "dong4_ba1"], | |
| "D0015": ["扬琴", "yang2_qin2"], | |
| "D0048": ["低音大锣", "Bass_da4_luo2"], | |
| "D0049": ["虎音锣", "hu3_yin1_luo2"], | |
| "D0050": ["小钹", "xiao3_bo1"], | |
| "D0051": ["钹", "bo1"], | |
| "D0058": ["提手(板)", "ti2_shou3_(ban3)"], | |
| "D0060": ["川小锣", "chuan1_xiao3_luo2"], | |
| "D0061": ["大铛铛", "da4_cheng1_cheng1"], | |
| "D0062": ["小铛铛", "xiao3_cheng1_cheng1"], | |
| "D0063": ["二馨", "er4_xin1"], | |
| "D0064": ["川大钵", "chuan1_da4_bo1"], | |
| "D0065": ["苏钵", "su1_bo1"], | |
| "D0066": ["川剧堂鼓", "chuan1_ju4_tang2_gu3"], | |
| "D0067": ["川铰", "chuan1_jiao3"], | |
| "D0068": ["川大锣", "chuan1_da4_luo2"], | |
| "D0069": ["蛮锣", "man2_luo2"], | |
| "D0070": ["包锣", "bao1_luo2"], | |
| "D0071": ["引鼓", "yin3_gu3"], | |
| "D0102": ["上杖鼓", "shang4_zhang4_gu3"], | |
| "D0103": ["小锣", "xiao3_luo2"], | |
| "D0104": ["圆锣", "yuan2_luo2"], | |
| "D0105": ["杖鼓", "zhang4_gu3"], | |
| "D0125": ["南鼓", "nan2_gu3"], | |
| "D0126": ["压脚鼓", "ya1_jiao3_gu3"], | |
| "D0127": ["钟", "zhong1"], | |
| "D0128": ["草锣", "cao3_luo2"], | |
| "D0129": ["锣仔", "luo2_zai3"], | |
| "D0130": ["响盏", "xiang3_zhan3"], | |
| "D0131": ["小叫", "xiao3_jiao4"], | |
| "D0132": ["拍", "pai1"], | |
| "D0137": ["渔鼓", "yu2_gu3"], | |
| "D0138": ["简板", "jian3_ban3"], | |
| "D0140": ["脚梆子", "jiao3_bang1_zi"], | |
| "D0143": ["双铃", "shuang1_ling2"], | |
| "D0144": ["小叫锣", "xiao3_jiao4_luo2"], | |
| "D0145": ["拍板", "pai1_ban3"], | |
| "D0146": ["四宝", "si4_bao3"], | |
| "D0147": ["响盏2", "xiang3_zhan3_2"], | |
| "D0172": ["碗碗", "wan3_wan3"], | |
| "D0173": ["代子", "dai4_zi"], | |
| "D0176": ["福(新)", "fu2_(reformed)"], | |
| "D0177": ["禄(新)", "lu4_(reformed)"], | |
| "D0178": ["寿(新)", "shou4_(reformed)"], | |
| "D0179": ["宜春三星鼓福鼓老鼓", "yi2_chun1_san1_xing1_gu3_fu2_gu3_(traditional)"], | |
| "D0180": ["宜春三星鼓禄鼓老鼓", "yi2_chun1_san1_xing1_gu3_lu4_gu3_(traditional)"], | |
| "D0181": ["宜春三星鼓寿鼓老鼓", "yi2_chun1_san1_xing1_gu3_shou4_gu3_(traditional)"], | |
| "D0184": ["宜春三星鼓双铛", "yi2_chun1_san1_xing1_gu3_shuang1_ding1"], | |
| "D0185": ["宜春三星鼓单铛", "yi2_chun1_san1_xing1_gu3_dan1_ding1"], | |
| "D0186": ["宜春三星鼓镲", "yi2_chun1_san1_xing1_gu3_chao3"], | |
| "D0241": ["编钟", "bian1_zhong1"], | |
| "D0242": ["编磬", "bian1_qing4"], | |
| "D0245": ["南梆子", "nan2_bang1_zi"], | |
| "D0246": ["北梆子", "bei3_bang1_zi"], | |
| "D0247": ["碰铃", "peng4_ling2"], | |
| "D0248": ["中国大鼓", "Chinese_da4_gu3"], | |
| "D0249": ["花盆鼓", "hua1_pen2_gu3"], | |
| "D0250": ["小堂鼓", "xiao3_tang2_gu3"], | |
| "D0251": ["扁鼓", "bian3_gu3"], | |
| "D0252": ["五音排鼓", "wu3_yin1_pai2_gu3"], | |
| "D0268": ["草帽镲", "cao3_mao4_chao3"], | |
| "D0269": ["铙", "nao2"], | |
| "D0270": ["铙钹", "nao2_bo1"], | |
| "D0271": ["小镲", "xiao3_chao3"], | |
| "D0272": ["抄锣", "chao1_luo2"], | |
| "D0273": ["中虎", "zhong1_hu3"], | |
| "D0274": ["武锣", "wu3_luo2"], | |
| "D0275": ["小锣2", "xiao3_luo2_2"], | |
| "D0276": ["马锣", "ma3_luo2"], | |
| "D0277": ["木鱼", "mu4_yu2"], | |
| "D0278": ["板鼓", "ban3_gu3"], | |
| "D0279": ["云锣", "yun2_luo2"], | |
| "D0284": ["斗锣", "dou3_luo2"], | |
| "D0286": ["曲锣", "qu3_luo2"], | |
| "D0287": ["深波", "shen1_bo1"], | |
| "D0290": ["大镲", "da4_chao3"], | |
| "D0298": ["编铓", "bian1_zhang1"], | |
| "D0299": ["牛铃", "niu2_ling2"], | |
| "D0315": ["竹排琴", "zhu2_pai2_qin2"], | |
| "D0325": ["那格拉", "na4_ge2_la1"], | |
| "D0326": ["库休克", "ku4_xiu1_ke4"], | |
| "D0327": ["萨巴依", "sa4_ba1_yi1"], | |
| "D0328": ["手鼓", "shou3_gu3"], | |
| "L0044": ["锡剧主胡", "xi1_ju4_zhu3_hu2"], | |
| "L0045": ["扬剧主胡", "yang2_ju4_zhu3_hu2"], | |
| "L0046": ["扬剧主胡F调", "yang2_ju4_zhu3_hu2_in_F"], | |
| "L0047": ["扬剧主胡(小西皮)", "yang2_ju4_zhu3_hu2_(xiao3_xi1_pi2)"], | |
| "L0053": ["广西彩调主胡", "guang3_xi1_cai3_diao4_zhu3_hu2"], | |
| "L0055": ["牛腿琴", "niu2_tui3_qin2"], | |
| "L0056": ["壮剧马骨胡D调", "zhuang4_ju4_ma3_gu3_hu2_in_D"], | |
| "L0072": ["盖板(新)D调", "gai4_ban3_(reformed)_in_D"], | |
| "L0073": ["盖板(传统)", "gai4_ban3_(traditional)"], | |
| "L0074": ["壮剧土胡", "zhuang4_ju4_tu3_hu2"], | |
| "L0075": ["晋剧晋胡", "jin4_ju4_jin4_hu2"], | |
| "L0076": ["壮剧土胡2", "zhuang4_ju4_tu3_hu2_2"], | |
| "L0077": ["晋剧二股弦", "jin4_ju4_er4_gu3_xian2"], | |
| "L0080": ["吕剧坠琴", "lv3_ju4_zhui4_qin2"], | |
| "L0084": ["奚琴(传统)", "xi1_qin2_(traditional)"], | |
| "L0085": ["奚琴(改良)", "xi1_qin2_(reformed)"], | |
| "L0086": ["中音奚琴(改良)", "Alto_xi1_qin2_(reformed)"], | |
| "L0115": ["莱芜梆子-梆胡", "lai2_wu2_bang1_zi-bang1_hu2"], | |
| "L0121": ["六角弦", "liu4_jiao3_xian2"], | |
| "L0122": ["壳仔弦", "ke2_zai3_xian2"], | |
| "L0133": ["陇剧陇胡(传统)", "long3_ju4_long3_hu2_(traditional)"], | |
| "L0134": ["陇剧陇胡(改良)D调", "long3_ju4_long3_hu2_(reformed)_in_D"], | |
| "L0135": ["齐琴", "qi2_qin2"], | |
| "L0136": ["渔胡", "yu2_hu2"], | |
| "L0139": ["坠胡", "zhui4_hu2"], | |
| "L0141": ["越胡", "yue4_hu2"], | |
| "L0148": ["板胡", "ban3_hu2"], | |
| "L0149": ["绍剧板胡", "shao4_ju4_ban3_hu2"], | |
| "L0150": ["宛梆子梆胡", "yuan1_bang1_zi_bang1_hu2"], | |
| "L0151": ["四弦", "si4_xian2"], | |
| "L0152": ["滇葫(小二胡)", "dian1_hu2_(xiao3_er4_hu2)"], | |
| "L0153": ["云南花灯丝弦", "yun2_nan2_hua1_deng1_si1_xian2"], | |
| "L0154": ["仕胡", "shi4_hu2"], | |
| "L0155": ["伬胡", "chi4_hu2"], | |
| "L0156": ["工胡", "gong1_hu2"], | |
| "L0157": ["大胡", "da4_hu2"], | |
| "L0158": ["低音伬胡", "Bass_chi4_hu2"], | |
| "L0160": ["丝弦", "si1_xian2"], | |
| "L0161": ["滇胡", "dian1_hu2"], | |
| "L0162": ["襄阳专用胡琴", "xiang1_yang2_zhuan1_yong4_hu2_qin2"], | |
| "L0163": ["雷胡", "lei2_hu2"], | |
| "L0164": ["赣胡", "gan4_hu2"], | |
| "L0165": ["高腔赣胡", "gao1_qiang1_gan4_hu2"], | |
| "L0166": ["高腔赣胡第2代", "gao1_qiang1_gan4_hu2_2nd_generation"], | |
| "L0167": ["黔胡", "qian2_hu2"], | |
| "L0168": ["花胡", "hua1_hu2"], | |
| "L0169": ["花胡2", "hua1_hu2_2"], | |
| "L0170": ["二股弦", "er4_gu3_xian2"], | |
| "L0239": ["高音板胡", "Treble_ban3_hu2"], | |
| "L0240": ["中音板胡", "Alto_ban3_hu2"], | |
| "L0256": ["雷琴", "lei2_qin2"], | |
| "L0266": ["二胡", "er4_hu2"], | |
| "L0285": ["二弦", "er4_xian2"], | |
| "L0288": ["椰胡", "ye1_hu2"], | |
| "L0291": ["扁八角高胡", "bian3_ba1_jiao3_gao1_hu2"], | |
| "L0292": ["六角高胡", "liu4_jiao3_gao1_hu2"], | |
| "L0297": ["中胡", "zhong1_hu2"], | |
| "L0307": ["芦笙", "lu2_sheng1"], | |
| "L0312": ["牛角胡", "niu2_jiao3_hu2"], | |
| "L0313": ["佤族独弦琴", "wa3_zu2_du2_xian2_qin2"], | |
| "L0314": ["葫芦琴", "hu2_lu2_qin2"], | |
| "T0006": ["陶布舒尔", "tao2_bu4_shu1_er3"], | |
| "T0007": ["雅托嘎", "ya3_tuo1_ga2"], | |
| "T0078": ["四股弦", "si4_gu3_xian2"], | |
| "T0081": ["玄琴", "xuan2_qin2"], | |
| "T0082": ["伽倻琴(改良)", "jia1_ye2_qin2_(reformed)"], | |
| "T0083": ["伽倻琴", "jia1_ye2_qin2"], | |
| "T0087": ["雅筝", "ya3_zheng1"], | |
| "T0088": ["扬琴2", "yang2_qin2_2"], | |
| "T0089": ["扬琴3", "yang2_qin2_3"], | |
| "T0111": ["三弦", "san1_xian2"], | |
| "T0116": ["八角月琴", "ba1_jiao3_yue4_qin2"], | |
| "T0159": ["双清", "shuang1_qing1"], | |
| "T0171": ["月琴", "yue4_qin2"], | |
| "T0238": ["大阮", "da4_ruan3"], | |
| "T0254": ["箜篌", "kong1_hou2"], | |
| "T0255": ["古筝", "gu3_zheng1"], | |
| "T0260": ["中阮", "zhong1_ruan3"], | |
| "T0261": ["柳琴", "liu3_qin2"], | |
| "T0262": ["琵琶", "pi2_pa2"], | |
| "T0267": ["扬琴4", "yang2_qin2_4"], | |
| "T0289": ["三弦2", "san1_xian2_2"], | |
| "T0294": ["南音琵琶", "nan2_yin1_pi2_pa2"], | |
| "T0295": ["南音三弦", "nan2_yin1_san1_xian2"], | |
| "T0300": ["澜沧小三弦", "lan2_cang1_xiao3_san1_xian2"], | |
| "T0301": ["玎", "ding1"], | |
| "T0302": ["傈傈族奇奔", "li4_li4_zu2_qi2_ben1"], | |
| "T0317": ["独弦琴", "du2_xian2_qin2"], | |
| "T0318": ["弹拨尔", "dan4_bo1_er3"], | |
| "T0319": ["低音热瓦普", "Bass_re4_wa3_pu3"], | |
| "T0320": ["民间热瓦普", "folk_re4_wa3_pu3"], | |
| "T0323": ["都它尔", "du1_ta1_er3"], | |
| } | |
| CLASSES = list(TRANSLATE.keys()) | |
| SAMPLE_RATE = 44100 | |
| def circular_padding(spec: np.ndarray, end: int): | |
| size = len(spec) | |
| if end <= size: | |
| return spec | |
| num_padding = end - size | |
| num_repeat = num_padding // size + int(num_padding % size != 0) | |
| padding = np.tile(spec, num_repeat) | |
| return np.concatenate((spec, padding))[:end] | |
| def wav2mel(audio_path: str, width=2, top_db=40): | |
| os.makedirs(TEMP_DIR, exist_ok=True) | |
| try: | |
| y, sr = librosa.load(audio_path, sr=SAMPLE_RATE) | |
| non_silents = librosa.effects.split(y, top_db=top_db) | |
| y = np.concatenate([y[start:end] for start, end in non_silents]) | |
| total_frames = len(y) | |
| if total_frames % (width * sr) != 0: | |
| count = total_frames // (width * sr) + 1 | |
| y = circular_padding(y, count * width * sr) | |
| mel_spec = librosa.feature.melspectrogram(y=y, sr=sr) | |
| log_mel_spec = librosa.power_to_db(mel_spec, ref=np.max) | |
| dur = librosa.get_duration(y=y, sr=sr) | |
| total_frames = log_mel_spec.shape[1] | |
| step = int(width * total_frames / dur) | |
| count = int(total_frames / step) | |
| begin = int(0.5 * (total_frames - count * step)) | |
| end = begin + step * count | |
| for i in range(begin, end, step): | |
| librosa.display.specshow(log_mel_spec[:, i : i + step]) | |
| plt.axis("off") | |
| plt.savefig( | |
| f"{TEMP_DIR}/{i}.jpg", | |
| bbox_inches="tight", | |
| pad_inches=0.0, | |
| ) | |
| plt.close() | |
| except Exception as e: | |
| print(f"Error converting {audio_path} : {e}") | |
| def wav2cqt(audio_path: str, width=2, top_db=40): | |
| os.makedirs(TEMP_DIR, exist_ok=True) | |
| try: | |
| y, sr = librosa.load(audio_path, sr=SAMPLE_RATE) | |
| non_silents = librosa.effects.split(y, top_db=top_db) | |
| y = np.concatenate([y[start:end] for start, end in non_silents]) | |
| total_frames = len(y) | |
| if total_frames % (width * sr) != 0: | |
| count = total_frames // (width * sr) + 1 | |
| y = circular_padding(y, count * width * sr) | |
| cqt_spec = librosa.cqt(y=y, sr=sr) | |
| log_cqt_spec = librosa.power_to_db(np.abs(cqt_spec) ** 2, ref=np.max) | |
| dur = librosa.get_duration(y=y, sr=sr) | |
| total_frames = log_cqt_spec.shape[1] | |
| step = int(width * total_frames / dur) | |
| count = int(total_frames / step) | |
| begin = int(0.5 * (total_frames - count * step)) | |
| end = begin + step * count | |
| for i in range(begin, end, step): | |
| librosa.display.specshow(log_cqt_spec[:, i : i + step]) | |
| plt.axis("off") | |
| plt.savefig( | |
| f"{TEMP_DIR}/{i}.jpg", | |
| bbox_inches="tight", | |
| pad_inches=0.0, | |
| ) | |
| plt.close() | |
| except Exception as e: | |
| print(f"Error converting {audio_path} : {e}") | |
| def wav2chroma(audio_path: str, width=2, top_db=40): | |
| os.makedirs(TEMP_DIR, exist_ok=True) | |
| try: | |
| y, sr = librosa.load(audio_path, sr=SAMPLE_RATE) | |
| non_silents = librosa.effects.split(y, top_db=top_db) | |
| y = np.concatenate([y[start:end] for start, end in non_silents]) | |
| total_frames = len(y) | |
| if total_frames % (width * sr) != 0: | |
| count = total_frames // (width * sr) + 1 | |
| y = circular_padding(y, count * width * sr) | |
| chroma_spec = librosa.feature.chroma_stft(y=y, sr=sr) | |
| log_chroma_spec = librosa.power_to_db(np.abs(chroma_spec) ** 2, ref=np.max) | |
| dur = librosa.get_duration(y=y, sr=sr) | |
| total_frames = log_chroma_spec.shape[1] | |
| step = int(width * total_frames / dur) | |
| count = int(total_frames / step) | |
| begin = int(0.5 * (total_frames - count * step)) | |
| end = begin + step * count | |
| for i in range(begin, end, step): | |
| librosa.display.specshow(log_chroma_spec[:, i : i + step]) | |
| plt.axis("off") | |
| plt.savefig( | |
| f"{TEMP_DIR}/{i}.jpg", | |
| bbox_inches="tight", | |
| pad_inches=0.0, | |
| ) | |
| plt.close() | |
| except Exception as e: | |
| print(f"Error converting {audio_path} : {e}") | |
| def most_frequent_value(lst: list): | |
| counter = Counter(lst) | |
| max_count = max(counter.values()) | |
| for element, count in counter.items(): | |
| if count == max_count: | |
| return element | |
| return None | |
| def infer(wav_path: str, log_name: str, folder_path=TEMP_DIR): | |
| if os.path.exists(folder_path): | |
| shutil.rmtree(folder_path) | |
| if not wav_path: | |
| return None, "请输入音频 Please input an audio!" | |
| try: | |
| model = EvalNet(log_name, len(TRANSLATE)).model | |
| except Exception as e: | |
| return None, f"{e}" | |
| spec = log_name.split("_")[-3] | |
| eval("wav2%s" % spec)(wav_path) | |
| jpgs = find_files(folder_path, ".jpg") | |
| preds = [] | |
| for jpg in jpgs: | |
| input = embed_img(jpg) | |
| output: torch.Tensor = model(input) | |
| preds.append(torch.max(output.data, 1)[1]) | |
| pred_id = most_frequent_value(preds) | |
| return ( | |
| os.path.basename(wav_path), | |
| f"{TRANSLATE[CLASSES[pred_id]][0]} ({TRANSLATE[CLASSES[pred_id]][1].capitalize()})", | |
| ) | |
| if __name__ == "__main__": | |
| warnings.filterwarnings("ignore") | |
| models = get_modelist() | |
| examples = [] | |
| example_wavs = find_files() | |
| model_num = len(models) | |
| for wav in example_wavs: | |
| examples.append([wav, models[random.randint(0, model_num - 1)]]) | |
| with gr.Blocks() as demo: | |
| gr.Interface( | |
| fn=infer, | |
| inputs=[ | |
| gr.Audio(label="上传录音 Upload a recording", type="filepath"), | |
| gr.Dropdown( | |
| choices=models, label="选择模型 Select a model", value=models[0] | |
| ), | |
| ], | |
| outputs=[ | |
| gr.Textbox(label="音频文件名 Audio filename", show_copy_button=True), | |
| gr.Textbox( | |
| label="中国乐器识别 Chinese instrument recognition", | |
| show_copy_button=True, | |
| ), | |
| ], | |
| examples=examples, | |
| cache_examples=False, | |
| flagging_mode="never", | |
| title="建议录音时长保持在 3s 左右<br>It is recommended to keep the recording length around 3s.", | |
| ) | |
| gr.Markdown( | |
| """ | |
| # 引用 Cite | |
| ```bibtex | |
| @dataset{zhaorui_liu_2021_5676893, | |
| author = {Monan Zhou, Shenyang Xu, Zhaorui Liu, Zhaowen Wang, Feng Yu, Wei Li and Baoqiang Han}, | |
| title = {CCMusic: an Open and Diverse Database for Chinese Music Information Retrieval Research}, | |
| month = {mar}, | |
| year = {2024}, | |
| publisher = {HuggingFace}, | |
| version = {1.2}, | |
| url = {https://huggingface.co/ccmusic-database} | |
| } | |
| ```""" | |
| ) | |
| demo.launch() | |