model

model.py

@@ -0,0 +1,165 @@
+import pandas as pd
+import numpy as np
+import os
+import shutil
+from datetime import datetime
+from timeit import default_timer as timer
+from utools import load_relevant_data_subset,mark_pred
+from utools import softmax
+import mediapipe as mp
+import cv2
+import json
+N=3
+
+ROWS_PER_FRAME=543
+with open('sign_to_prediction_index_map_cn.json', 'r') as f:
+    person_dict = json.load(f)
+inverse_dict=dict([val,key] for key,val in person_dict.items())
+
+
+def r_holistic(video_path):
+    mp_drawing = mp.solutions.drawing_utils
+    mp_drawing_styles = mp.solutions.drawing_styles
+    mp_holistic = mp.solutions.holistic
+    frame_number = 0
+    frame = []
+    type_ = []
+    index = []
+    x = []
+    y = []
+    z = []
+    cap=cv2.VideoCapture(video_path)
+    frame_width = int(cap.get(3))
+    frame_height = int(cap.get(4))
+    fps = int(cap.get(cv2.CAP_PROP_FPS))
+    frame_size = (frame_width, frame_height)
+    fourcc = cv2.VideoWriter_fourcc(*"VP80") #cv2.VideoWriter_fourcc('H.264')
+    output_video = "output_recorded_holistic.webm"
+    out = cv2.VideoWriter(output_video, fourcc, int(fps/N), frame_size)
+    with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:
+        n=0
+        while cap.isOpened():
+            frame_number+=1
+            n+=1
+            ret, image = cap.read()
+            if not ret:
+                break
+            if n%N==0:
+                image.flags.writeable = False
+                image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
+                #mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=RGB_frame)
+                results = holistic.process(image)
+
+                # Draw landmark annotation on the image.
+                image.flags.writeable = True
+                image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+                mp_drawing.draw_landmarks(
+                    image,
+                    results.face_landmarks,
+                    mp_holistic.FACEMESH_CONTOURS,
+                    landmark_drawing_spec=None,
+                    connection_drawing_spec=mp_drawing_styles
+                    .get_default_face_mesh_contours_style())
+                mp_drawing.draw_landmarks(
+                    image,
+                    results.pose_landmarks,
+                    mp_holistic.POSE_CONNECTIONS,
+                    landmark_drawing_spec=mp_drawing_styles
+                    .get_default_pose_landmarks_style())
+                # Flip the image horizontally for a selfie-view display.
+                #if cv2.waitKey(5) & 0xFF == 27:
+                out.write(image)
+                
+                if(results.face_landmarks is None):
+                    for i in range(468):
+                        frame.append(frame_number)
+                        type_.append("face")
+                        index.append(ind)
+                        x.append(None)
+                        y.append(None)
+                        z.append(None)
+                else:
+                    for ind,val in enumerate(results.face_landmarks.landmark):
+                        frame.append(frame_number)
+                        type_.append("face")
+                        index.append(ind)
+                        x.append(val.x)
+                        y.append(val.y)
+                        z.append(val.z)
+                #left hand
+                if(results.left_hand_landmarks is None):
+                    for i in range(21):
+                        frame.append(frame_number)
+                        type_.append("left_hand")
+                        index.append(ind)
+                        x.append(None)
+                        y.append(None)
+                        z.append(None)
+                else:
+                    for ind,val in enumerate(results.left_hand_landmarks.landmark):
+                        frame.append(frame_number)
+                        type_.append("left_hand")
+                        index.append(ind)
+                        x.append(val.x)
+                        y.append(val.y)
+                        z.append(val.z)
+                #pose
+                if(results.pose_landmarks is None):
+                    for i in range(33):
+                        frame.append(frame_number)
+                        type_.append("pose")
+                        index.append(ind)
+                        x.append(None)
+                        y.append(None)
+                        z.append(None)
+                else:
+                    for ind,val in enumerate(results.pose_landmarks.landmark):
+                        frame.append(frame_number)
+                        type_.append("pose")
+                        index.append(ind)
+                        x.append(val.x)
+                        y.append(val.y)
+                        z.append(val.z)
+                #right hand
+                if(results.right_hand_landmarks is None):
+                    for i in range(21):
+                        frame.append(frame_number)
+                        type_.append("right_hand")
+                        index.append(ind)
+                        x.append(None)
+                        y.append(None)
+                        z.append(None)
+                else:
+                    for ind,val in enumerate(results.right_hand_landmarks.landmark):
+                        frame.append(frame_number)
+                        type_.append("right_hand")
+                        index.append(ind)
+                        x.append(val.x)
+                        y.append(val.y)
+                        z.append(val.z)
+        #break
+    cap.release()
+    out.release()
+    cv2.destroyAllWindows()
+    df1 = pd.DataFrame({
+            "frame" : frame,
+            "type"  : type_,
+            "landmark_index" : index,
+            "x" : x,
+            "y" : y,
+            "z" : z
+        })
+    aa=load_relevant_data_subset(df1)
+    model_path_1='model_1.tflite'
+    model_path_2='model_2.tflite'
+    model_path_3='model_3.tflite'
+    #interpreter = tflite.Interpreter(model_path_1)
+    #found_signatures = list(interpreter.get_signature_list().keys())
+    #prediction_fn = interpreter.get_signature_runner("serving_default")
+    output_1 = mark_pred(model_path_1,aa)
+    output_2 = mark_pred(model_path_2,aa)
+    output_3 = mark_pred(model_path_3,aa)
+    output=softmax(output_1['outputs'])+softmax(output_2['outputs'])+softmax(output_3['outputs'])
+    sign = output.argmax()
+    lb = inverse_dict.get(sign)
+    yield output_video,lb