{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overwriting utools.py\n" ] } ], "source": [ "%%writefile utools.py\n", "import tflite_runtime.interpreter as tflite \n", "import tflite_runtime\n", "import numpy as np\n", "ROWS_PER_FRAME=543\n", "def load_relevant_data_subset(df):\n", " data_columns = ['x', 'y', 'z']\n", " data=df[data_columns]\n", " n_frames = int(len(data) / ROWS_PER_FRAME)#单个文件的总帧数\n", " data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n", " return data.astype(np.float32)\n", "\n", "def mark_pred(model_path_1,aa):\n", " interpreter = tflite.Interpreter(model_path_1)\n", " found_signatures = list(interpreter.get_signature_list().keys())\n", " prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n", " output_1 = prediction_fn(inputs=aa)\n", " return output_1\n", "\n", "def softmax(x, axis=None):\n", " x_exp = np.exp(x - np.max(x, axis=axis, keepdims=True))\n", " return x_exp / np.sum(x_exp, axis=axis, keepdims=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overwriting model.py\n" ] } ], "source": [ "%%writefile model.py\n", "import pandas as pd\n", "import numpy as np\n", "import os\n", "import shutil\n", "from datetime import datetime\n", "from timeit import default_timer as timer\n", "from utools import load_relevant_data_subset,mark_pred\n", "from utools import softmax\n", "import mediapipe as mp\n", "import cv2\n", "import json\n", "N=3\n", "\n", "ROWS_PER_FRAME=543\n", "with open('sign_to_prediction_index_map_cn.json', 'r') as f:\n", " person_dict = json.load(f)\n", "inverse_dict=dict([val,key] for key,val in person_dict.items())\n", "\n", "\n", "def r_holistic(video_path):\n", " mp_drawing = mp.solutions.drawing_utils\n", " mp_drawing_styles = mp.solutions.drawing_styles\n", " mp_holistic = mp.solutions.holistic\n", " frame_number = 0\n", " frame = []\n", " type_ = []\n", " index = []\n", " x = []\n", " y = []\n", " z = []\n", " cap=cv2.VideoCapture(video_path)\n", " frame_width = int(cap.get(3))\n", " frame_height = int(cap.get(4))\n", " fps = int(cap.get(cv2.CAP_PROP_FPS))\n", " frame_size = (frame_width, frame_height)\n", " fourcc = cv2.VideoWriter_fourcc(*\"VP80\") #cv2.VideoWriter_fourcc('H.264')\n", " output_video = \"output_recorded_holistic.webm\"\n", " out = cv2.VideoWriter(output_video, fourcc, int(fps/N), frame_size)\n", " with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:\n", " n=0\n", " while cap.isOpened():\n", " frame_number+=1\n", " n+=1\n", " ret, image = cap.read()\n", " if not ret:\n", " break\n", " if n%N==0:\n", " image.flags.writeable = False\n", " image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n", " #mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=RGB_frame)\n", " results = holistic.process(image)\n", "\n", " # Draw landmark annotation on the image.\n", " image.flags.writeable = True\n", " image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n", " mp_drawing.draw_landmarks(\n", " image,\n", " results.face_landmarks,\n", " mp_holistic.FACEMESH_CONTOURS,\n", " landmark_drawing_spec=None,\n", " connection_drawing_spec=mp_drawing_styles\n", " .get_default_face_mesh_contours_style())\n", " mp_drawing.draw_landmarks(\n", " image,\n", " results.pose_landmarks,\n", " mp_holistic.POSE_CONNECTIONS,\n", " landmark_drawing_spec=mp_drawing_styles\n", " .get_default_pose_landmarks_style())\n", " # Flip the image horizontally for a selfie-view display.\n", " #if cv2.waitKey(5) & 0xFF == 27:\n", " out.write(image)\n", " \n", " if(results.face_landmarks is None):\n", " for i in range(468):\n", " frame.append(frame_number)\n", " type_.append(\"face\")\n", " index.append(ind)\n", " x.append(None)\n", " y.append(None)\n", " z.append(None)\n", " else:\n", " for ind,val in enumerate(results.face_landmarks.landmark):\n", " frame.append(frame_number)\n", " type_.append(\"face\")\n", " index.append(ind)\n", " x.append(val.x)\n", " y.append(val.y)\n", " z.append(val.z)\n", " #left hand\n", " if(results.left_hand_landmarks is None):\n", " for i in range(21):\n", " frame.append(frame_number)\n", " type_.append(\"left_hand\")\n", " index.append(ind)\n", " x.append(None)\n", " y.append(None)\n", " z.append(None)\n", " else:\n", " for ind,val in enumerate(results.left_hand_landmarks.landmark):\n", " frame.append(frame_number)\n", " type_.append(\"left_hand\")\n", " index.append(ind)\n", " x.append(val.x)\n", " y.append(val.y)\n", " z.append(val.z)\n", " #pose\n", " if(results.pose_landmarks is None):\n", " for i in range(33):\n", " frame.append(frame_number)\n", " type_.append(\"pose\")\n", " index.append(ind)\n", " x.append(None)\n", " y.append(None)\n", " z.append(None)\n", " else:\n", " for ind,val in enumerate(results.pose_landmarks.landmark):\n", " frame.append(frame_number)\n", " type_.append(\"pose\")\n", " index.append(ind)\n", " x.append(val.x)\n", " y.append(val.y)\n", " z.append(val.z)\n", " #right hand\n", " if(results.right_hand_landmarks is None):\n", " for i in range(21):\n", " frame.append(frame_number)\n", " type_.append(\"right_hand\")\n", " index.append(ind)\n", " x.append(None)\n", " y.append(None)\n", " z.append(None)\n", " else:\n", " for ind,val in enumerate(results.right_hand_landmarks.landmark):\n", " frame.append(frame_number)\n", " type_.append(\"right_hand\")\n", " index.append(ind)\n", " x.append(val.x)\n", " y.append(val.y)\n", " z.append(val.z)\n", " #break\n", " cap.release()\n", " out.release()\n", " cv2.destroyAllWindows()\n", " df1 = pd.DataFrame({\n", " \"frame\" : frame,\n", " \"type\" : type_,\n", " \"landmark_index\" : index,\n", " \"x\" : x,\n", " \"y\" : y,\n", " \"z\" : z\n", " })\n", " aa=load_relevant_data_subset(df1)\n", " model_path_1='model_1.tflite'\n", " model_path_2='model_2.tflite'\n", " model_path_3='model_3.tflite'\n", " #interpreter = tflite.Interpreter(model_path_1)\n", " #found_signatures = list(interpreter.get_signature_list().keys())\n", " #prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n", " output_1 = mark_pred(model_path_1,aa)\n", " output_2 = mark_pred(model_path_2,aa)\n", " output_3 = mark_pred(model_path_3,aa)\n", " output=softmax(output_1['outputs'])+softmax(output_2['outputs'])+softmax(output_3['outputs'])\n", " sign = output.argmax()\n", " lb = inverse_dict.get(sign)\n", " yield output_video,lb" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overwriting app.py\n" ] } ], "source": [ "%%writefile app.py\n", "\n", "import gradio as gr\n", "from model import r_holistic\n", "\n", "title='手语动作分类'\n", "description = \"此分类模型可以识别250个[ASL](https://www.lifeprint.com/)手语动作\\\n", " 并将其转化为特定的标签, 标签列表见链接[sign_to_prediction_index_map.json](sign_to_prediction_index_map.json), \\\n", " 大家可以使用示例视频进行测试, 也可以根据列表下载或模拟相应的手语视频测试输出.\\\n", " \\n工作流程:\\\n", " \\n 1. landmark提取, 我使用了[ MediaPipe Holistic Solution](https://ai.google.dev/edge/mediapipe/solutions/vision/holistic_landmarker)进行landmark提取.\\\n", " \\n 2. 利用landmark进行手语识别, 我使用了自己搭建并训练的模型, 主体框架为cnn和transform，此模型在测试数据集上精度在90%以上.\"\n", "\n", "output_video_file = gr.Video(label=\"landmark输出\")\n", "output_text=gr.Textbox(label=\"手语预测结果\")\n", "slider_1=gr.Slider(0,1,label='detection_confidence')\n", "slider_2=gr.Slider(0,1,label='tracking_confidence')\n", "\n", "iface = gr.Interface(\n", " fn=r_holistic,\n", " inputs=[gr.Video(sources=None, label=\"手语视频片段\")],\n", " outputs= [output_video_file,output_text],\n", " title=title, \n", " description=description,\n", " examples=['book.mp4','book2.mp4','chair1.mp4','chair2.mp4'],\n", " #cache_examples=True,\n", " ) #[\"hand-land-mark-video/01.mp4\",\"hand-land-mark-video/02.mp4\"]\n", " \n", "\n", "iface.launch(share=True)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "myenv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" } }, "nbformat": 4, "nbformat_minor": 2 }