基于Mediapipe的手势控制赛车游戏程序
import vgamepad as vg

gamepad = vg.VX360Gamepad()


def gamepad_Control(signal):
    gamepad.reset()
    angle = signal["Angle"]
    if signal["up"] and not signal["bark"]:
        gamepad.right_trigger_float(value_float=0.7)
        gamepad.left_joystick_float(x_value_float=angle, y_value_float=0.5)
        # print("前进")
    elif signal["bark"]:
        gamepad.left_trigger_float(value_float=0.8)
        gamepad.left_joystick_float(x_value_float=angle, y_value_float=-0.5)
        # print("后退")
    elif not signal["up"] and not signal["bark"]:
        gamepad.left_joystick_float(x_value_float=angle, y_value_float=0.5)

    gamepad.update()

    return
con_signal.py
xiaochouyulalala/Handler
import cv2
import mediapipe as mp

class GestureDetector():

    def __init__(self, mode=False, maxHands=2, detectionCon=0.5, trackCon=0.5):
        
        self.mode = mode
        self.maxHands = maxHands
        self.detectionCon = detectionCon
        self.trackCon = trackCon

        self.mpHands = mp.solutions.hands
        self.hands = self.mpHands.Hands(static_image_mode=self.mode, 
                                            max_num_hands=self.maxHands,
                                            min_detection_confidence=self.detectionCon, 
                                            min_tracking_confidence=self.trackCon)
        self.mpDraw = mp.solutions.drawing_utils
        
    def findHands(self, img, draw=True):
        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        self.x,self.y,c=img.shape
        self.results = self.hands.process(imgRGB)
        if self.results.multi_hand_landmarks:
            for handLms in self.results.multi_hand_landmarks:
                if draw:
                    self.mpDraw.draw_landmarks(imgRGB, handLms, self.mpHands.HAND_CONNECTIONS)
        return imgRGB

    def findPosition(self, img, handNo=0):
        lmList = []
        if self.results.multi_hand_landmarks:
            myHand = self.results.multi_hand_landmarks[handNo]
            for id, lm in enumerate(myHand.landmark):
                h, w, c = img.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                lmList.append([id, cx, cy])
        return lmList
gesture_detector.py
xiaochouyulalala/Handler
from PySide2.QtWidgets import QWidget, QApplication, QSlider, QLabel, QPushButton, QHBoxLayout, QVBoxLayout
from PySide2.QtCore import Qt, QTimer,QRect
from PySide2.QtGui import QPainter, QPixmap,QFont,QImage
import sys
import cv2
import utils
from gesture_detector import GestureDetector
import con_signal
from collections import defaultdict
from pykalman import KalmanFilter
import math

# 创建一个摄像头捕获对象，使用默认的摄像头设备
cap = cv2.VideoCapture(0)
detecor = GestureDetector()

# 创建一个覆盖窗口类，用来显示手势反馈和摄像头画面
class OverlayWidget(QWidget):

    def __init__(self):
        QWidget.__init__(self, geometry=QApplication.desktop().screenGeometry())

        self.setAttribute(Qt.WA_TranslucentBackground)
        self.setWindowFlags(Qt.FramelessWindowHint | Qt.WindowStaysOnTopHint)
        self.frame = None
        # 创建一个定时器，用来定时更新窗口的内容
        timer = QTimer(self)
        timer.timeout.connect(self.update)
        timer.start(10)
        self.cmd_history = []
        self.cmd_history_max_len = 5

        # 初始化手势图像和文字的字典，用来存储不同手势对应的图像和文字
        self.gesture_img_dict = {
            "up": QPixmap("./source/speedup.png"),
            "left": QPixmap("./source/left.png"),
            "right": QPixmap("./source/right.png"),
            "bark": QPixmap("./source/bark.png"),
            "unknown": QPixmap("unknow.png")
        }
        self.gesture_text_dict = {
            "up": "加速",
            "bark": "刹车",
            "left": "左转",
            "right": "右转",
            "unknown": "未知"
        }

        # 初始化当前手势图像和文字为未知
        self.gesture_img = self.gesture_img_dict["unknown"]
        self.gesture_text = self.gesture_text_dict["unknown"]

        self.kf = KalmanFilter(transition_matrices=1, observation_matrices=1)
  
        # kalman
        self.kf.initial_state_mean = 0 
        self.kf.initial_state_covariance = 1

        self.kf.transition_covariance = 0.001
        self.kf.observation_covariance = 0.5

        self.observation = 1 

    def get_most_frequent(self):  
        count = defaultdict(int)  
        for cmd_str in self.cmd_history:
            count[cmd_str] += 1
        most_common = max(count, key=count.get)
        self.cmd_history=[]
        return  most_common

    def stable_sigmoid(self,x):
        a = 0.05
        if x >= 0:
            z = math.exp(-a*x)
            sig = 1 / (1 + z)
            return sig-0.5
        else:
            z = math.exp(a*x)
            sig = z / (1 + z)
            return sig-0.5

    def update(self):
        
        ret, frame = cap.read()
        if not ret:
            return
        frame= cv2.flip(frame,1)

        self.frame = detecor.findHands(frame)
        detecor.findPosition(self.frame)
        gesture = utils.update_gesture_status_low(detecor)

        cmd1='unknown'
        cmd2='unknown'
        
        self.kf.transition_matrices = 1
        self.kf.observation_matrices = self.observation
        filtered_angle = self.kf.filter(gesture['Angle'])[0][0][0]
        filtered_angle = self.stable_sigmoid(filtered_angle)
        filtered_angle = filtered_angle*2
        print(filtered_angle)
        signal = {
            "up":False,
            "bark":False,
            "Angle":filtered_angle
        }

        if(gesture['Left Thumb']):
            cmd2 = 'bark'
            signal["bark"] = True
        elif(gesture['Right Thumb']):
            cmd2 = 'up'
            signal["up"] = True
        if(filtered_angle<-0.25):
            cmd1 = 'left'
        elif(filtered_angle>0.25):
            cmd1 = 'right'

        self.cmd_history.append(cmd1 + '_' + cmd2)
        
        if len(self.cmd_history) > self.cmd_history_max_len:

            gmf = self.get_most_frequent()
            
            cmd1, cmd2 = gmf.split('_')

            # 根据手势类别，从字典中获取对应的图像和文字
            if(cmd1!='unknown'):
                self.gesture_img = self.gesture_img_dict.get(cmd1, self.gesture_img_dict["unknown"])
                self.gesture_text = self.gesture_text_dict.get(cmd1, self.gesture_text_dict["unknown"])
            elif(cmd2!='unknown'):
                self.gesture_img = self.gesture_img_dict.get(cmd2, self.gesture_img_dict["unknown"])
                self.gesture_text = self.gesture_text_dict.get(cmd2, self.gesture_text_dict["unknown"])
            else:
                self.gesture_img = self.gesture_img_dict.get('unknow', self.gesture_img_dict["unknown"])
                self.gesture_text = self.gesture_text_dict.get('unknow', self.gesture_text_dict["unknown"])
            print(f"Detected gesture: {gesture}")
            con_signal.gamepad_Control(signal)
            # 更新窗口的绘制
        self.repaint()
    
    def paintEvent(self, event):
        painter = QPainter(self)

        opacity = 0.8
        painter.setOpacity(opacity)
        
        rect = self.gesture_img.rect()
        rect.moveCenter(self.rect().center())
        painter.drawPixmap(rect.topLeft(), self.gesture_img)

        
        painter.setFont(QFont("Arial", 20))
        painter.drawText(self.rect(), Qt.AlignBottom | Qt.AlignHCenter, self.gesture_text)

        img = QImage(self.frame.data, self.frame.shape[1], self.frame.shape[0], 
                    self.frame.strides[0], QImage.Format_RGB888)
                    
        img_width = img.width() // 2
        img_height = img.height() // 2
        camera_rect = QRect(0, 0, 
                            img_width, img_height)
                            
        # 绘制图像                
        painter.drawImage(camera_rect, img)

# 创建一个主窗口类，用来显示灵敏度调节和退出按钮
class MainWindow(QWidget):

    def __init__(self):
        QWidget.__init__(self)

        # 创建一个滑动条用来调节手势识别的灵敏度
        self.sensitivity_slider = QSlider(Qt.Horizontal)
        self.sensitivity_slider.setMinimum(0)
        self.sensitivity_slider.setMaximum(100)
        self.sensitivity_slider.setValue(50) # 默认值为50
        self.sensitivity_slider.valueChanged.connect(self.on_sensitivity_changed)

        # 标签
        self.sensitivity_label = QLabel("灵敏度: 50")

        # 按钮
        self.exit_button = QPushButton("退出")
        self.exit_button.clicked.connect(self.on_exit_clicked)

        self.hbox = QHBoxLayout()
        self.hbox.addWidget(self.sensitivity_slider)
        self.hbox.addWidget(self.sensitivity_label)

        self.vbox = QVBoxLayout()
        self.vbox.addLayout(self.hbox)
        self.vbox.addWidget(self.exit_button)

        self.setLayout(self.vbox)

    def on_sensitivity_changed(self, value):
        self.sensitivity_label.setText(f"灵敏度: {value}")
        

    def on_exit_clicked(self):
        # 关闭程序
        sys.exit()
    

if __name__ == "__main__":
   app = QApplication([]) 
   overlay_window = OverlayWidget()
   overlay_window.show()
   main_window = MainWindow()
   main_window.show()
   app.exec_()

main.py
xiaochouyulalala/Handler
import cv2
import utils
from gesture_detector import GestureDetector

gesture_status = {
    "Left Fist": False, 
    "Right Fist": False,
    "Left Thumb": False,
    "Right Thumb": False,
    "Angle": 0  
}
def show_gesture_status(img, gesture_status):

    font = cv2.FONT_HERSHEY_PLAIN
  
    cv2.putText(img, "Left Fist: " + str(gesture_status["Left Fist"]), 
    (10, 20), font, 1, (0,255,0), 2)

    cv2.putText(img, "Right Fist: " + str(gesture_status["Right Fist"]),
    (10, 50), font, 1, (0,255,0), 2)   

    cv2.putText(img, "Left Thumb: " + str(gesture_status["Left Thumb"]), 
    (10, 70), font, 1, (0,255,0), 2)

    cv2.putText(img, "Right Thumb: " + str(gesture_status["Right Thumb"]),
    (10, 90), font, 1, (0,255,0), 2) 
    cv2.putText(img, "Angle: " + str(gesture_status["Angle"]), 
    (10, 110), font, 1, (0,255,0), 2)

def main():
    cap = cv2.VideoCapture(0)
    detecor = GestureDetector()
    while True:
        success, img = cap.read()
        img= cv2.flip(img,1)

        img = detecor.findHands(img)
        detecor.findPosition(img)
        
        gesture_status = utils.update_gesture_status_low(detecor)
        # print(gesture_status)
        # signal(gesture_status)
        show_gesture_status(img, gesture_status)
        cv2.imshow('img',img)
        cv2.waitKey(0)


if __name__ == "__main__":
    main()
test.py
xiaochouyulalala/Handler
import math
import mediapipe as mp

mp_hands = mp.solutions.hands
 

# def GestureRecognizer(img):
#     model_path = 'keras_model.h5'
#     BaseOptions = mp.tasks.BaseOptions
#     GestureRecognizer = mp.tasks.vision.GestureRecognizer
#     GestureRecognizerOptions = mp.tasks.vision.GestureRecognizerOptions
#     VisionRunningMode = mp.tasks.vision.RunningMode


#     options = GestureRecognizerOptions(
#         base_options=BaseOptions(model_asset_path=model_path),
#         num_hands = 2,
#         running_mode=VisionRunningMode.IMAGE)

#     with GestureRecognizer.create_from_options(options) as recognizer:
#         recognition_result = recognizer.recognize(img)
#         return recognition_result

def calculateAngle(results):
    # 计算两个拳头的夹角

    handedness_list = results.multi_handedness 
    handedness = handedness_list[0]
    if(handedness=='Left'):
        left_fist = results.multi_hand_landmarks[0]
        right_fist = results.multi_hand_landmarks[1]
    else:
        left_fist = results.multi_hand_landmarks[1]
        right_fist = results.multi_hand_landmarks[0]

    x1, y1 = vector_from_landmarks(left_fist) 
    x2, y2 = vector_from_landmarks(right_fist)

    angle = math.atan((y1-y2)/(x1-x2))
    angle = math.degrees(angle)
    return angle


def vector_from_landmarks(landmarks):
    # 从手部关键点获取向量
    x = (landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x + landmarks.landmark[mp_hands.HandLandmark.WRIST].x)/2
    y = (landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y + landmarks.landmark[mp_hands.HandLandmark.WRIST].y)/2
    return x, y

def vector_2d_angle(v1,v2):

    v1_x=v1[0]
    v1_y=v1[1]
    v2_x=v2[0]
    v2_y=v2[1]
    try:
        angle_= math.degrees(math.acos((v1_x*v2_x+v1_y*v2_y)/(((v1_x**2+v1_y**2)**0.5)*((v2_x**2+v2_y**2)**0.5))))
    except:
        angle_ = 1234.
    if angle_ > 180.:
        angle_ = 1234.
    return angle_

def hand_angle(hand_):

    angle_list = []
    angle_ = vector_2d_angle(
        ((int(hand_[0][0])- int(hand_[2][0])),(int(hand_[0][1])-int(hand_[2][1]))),
        ((int(hand_[3][0])- int(hand_[4][0])),(int(hand_[3][1])- int(hand_[4][1])))
        )
    angle_list.append(angle_)
    angle_ = vector_2d_angle(
        ((int(hand_[0][0])-int(hand_[6][0])),(int(hand_[0][1])- int(hand_[6][1]))),
        ((int(hand_[7][0])- int(hand_[8][0])),(int(hand_[7][1])- int(hand_[8][1])))
        )
    angle_list.append(angle_)
    angle_ = vector_2d_angle(
        ((int(hand_[0][0])- int(hand_[10][0])),(int(hand_[0][1])- int(hand_[10][1]))),
        ((int(hand_[11][0])- int(hand_[12][0])),(int(hand_[11][1])- int(hand_[12][1])))
        )
    angle_list.append(angle_)
    angle_ = vector_2d_angle(
        ((int(hand_[0][0])- int(hand_[14][0])),(int(hand_[0][1])- int(hand_[14][1]))),
        ((int(hand_[15][0])- int(hand_[16][0])),(int(hand_[15][1])- int(hand_[16][1])))
        )
    angle_list.append(angle_)
    angle_ = vector_2d_angle(
        ((int(hand_[0][0])- int(hand_[18][0])),(int(hand_[0][1])- int(hand_[18][1]))),
        ((int(hand_[19][0])- int(hand_[20][0])),(int(hand_[19][1])- int(hand_[20][1])))
        )
    angle_list.append(angle_)
    return angle_list

def check_fist(angle_list):
    thr_angle = 65.
    thr_angle_thumb = 53.

    if 1234. not in angle_list:
        if (angle_list[0]>thr_angle_thumb) and (angle_list[1]>thr_angle) and (angle_list[2]>thr_angle) and (angle_list[3]>thr_angle) and (angle_list[4]>thr_angle):
            return True
    return False

def check_thumb(angle_list):
    thr_angle = 65.
    thr_angle_thumb = 25.
    if 1234. not in angle_list:
        if (angle_list[0]<thr_angle_thumb)  and (angle_list[1]>thr_angle) and (angle_list[2]>thr_angle) and (angle_list[3]>thr_angle) and (angle_list[4]>thr_angle):
            return True
    return False

def update_gesture_status_low(detector):
    results = detector.results
    
    left_fist = False
    right_fist = False 
    left_thumb = False
    right_thumb = False
    angle = 0

    if results.multi_hand_landmarks:
        
        for idx,hand_landmarks in enumerate(results.multi_hand_landmarks):
            handedness = results.multi_handedness[idx].classification[0].label
            # print(handedness)
            hand_local = []
            for i in range(21):
                x = hand_landmarks.landmark[i].x*detector.y
                y = hand_landmarks.landmark[i].y*detector.x
                hand_local.append((x,y))
            if hand_local:
                angle_list = hand_angle(hand_local)
            if handedness == 'Right':
                right_fist = check_fist(angle_list)
                if(not right_fist):
                    right_thumb = check_thumb(angle_list)
            
            elif handedness == 'Left':
                left_fist = check_fist(angle_list)
                if(not left_fist):
                    left_thumb = check_thumb(angle_list)

    if ((left_fist == False) + (right_fist == False) + (left_thumb == False) + (right_thumb == False) < 3):
        angle = calculateAngle(results)
    return {
            "Left Fist": left_fist, 
            "Right Fist": right_fist,
            "Left Thumb": left_thumb,
            "Right Thumb": right_thumb,
            "Angle": angle 
            }

# 方案2
# def update_gesture_status(img):

#     left_fist = False
#     right_fist = False
#     left_thumb = False
#     right_thumb = False
#     angle = 0

#     results = GestureRecognizer(img)
#     hand_landmarks_list = results.hand_landmarks
#     handedness_list = results.handedness
#     gestures_list = results.gestures
#     for idx in range(len(hand_landmarks_list)):
#         # hand_landmarks = hand_landmarks_list[idx]
#         handedness = handedness_list[idx].category_name
#         gestures = gestures_list[idx].category_name
#         if(handedness=='Left' and gestures == 'fist'):
#             left_fist = True
#         elif(handedness=='Left' and gestures == 'thumb'):
#             left_thumb = True
#         elif(handedness=='Right' and gestures == 'fist'):
#             right_fist = True
#         elif(handedness=='Right' and gestures == 'thumb'):
#             right_thumb = True
    

#     if ((left_fist == False) + (right_fist == False) + (left_thumb == False) + (right_thumb == False) < 3):
#         angle = calculateAngle(results)
#     return {
#             "Left Fist": left_fist, 
#             "Right Fist": right_fist,
#             "Left Thumb": left_thumb,
#             "Right Thumb": right_thumb,
#             "Angle": angle 
#             }

utils.py
xiaochouyulalala/Handler