Mediapipe 手掌侦测 ( 新 )

MediaPipe 的 Hand Landmark Detection 可以在检测双手的手掌，再透过 OpenCV 读取摄影镜头图像进行辨识，在手掌与每只手指标记骨架。

快速导览：

• 侦测手掌信息
• 即时绘制手掌骨架

因为程序使用 Jupyter 搭配 Tensorflow 进行开发，所以请先阅读“使用 Anaconda”和“使用 MediaPipe”，安装对应的软件包，如果不要使用 Juputer，也可参考“使用 Python 虚拟环境”，建立虚拟环境进行实作。

侦测手掌信息

MediaPipe Hands 利用多个模型协同工作，可以侦测手掌模型，返回手掌与每只手指精确的 3D 关键点，MediaPipe Hand 除了可以侦测清晰的手掌形状与动作，更可以判断出被少部分被遮蔽的手指形状和动作，再清晰的画面下，针对手掌判断的精准度可达 95.7%。

Mediapipe 侦测手掌后，会在手掌与手指上产生 21 个具有 x、y、z 座标的节点，透过包含立体深度的节点，就能在 3D 场景中做出多种不同的应用，下图标示出每个节点的顺序和位置 ( 图片来源 )。

下方的程序码延伸“读取并播放影片”文章的范例，搭配 mediapipe 手掌侦测的方法，透过摄影镜头获取图像后，即时显示手掌的信息。

import cv2
import mediapipe as mp

BaseOptions = mp.tasks.BaseOptions
HandLandmarker = mp.tasks.vision.HandLandmarker
HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
VisionRunningMode = mp.tasks.vision.RunningMode

# 偵測手掌設定
options = HandLandmarkerOptions(
    num_hands=2,
    base_options=BaseOptions(model_asset_path='model/hand_landmarker.task'),
    running_mode=VisionRunningMode.IMAGE)

with HandLandmarker.create_from_options(options) as landmarker:
    cap = cv2.VideoCapture(0)   
    if not cap.isOpened():
        print("Cannot open camera")
        exit()
    while True:
        ret, frame = cap.read()
        w = frame.shape[1]                  # 畫面寬度
        h = frame.shape[0]                  # 畫面高度
        if not ret:
            print("Cannot receive frame")
            break

        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
        hand_landmarker_result = landmarker.detect(mp_image)
        
        print(hand_landmarker_result)
        
        cv2.imshow('oxxostudio', frame)
        if cv2.waitKey(5) == ord('q'):
            break    # 按下 q 鍵停止
cap.release()
cv2.destroyAllWindows()

即时绘制手掌骨架

侦测到手掌后，参考“Hand Landmarks Detection with MediaPipe Tasks”范例程序码，加入标记绘图的函数区块，就可以在侦测到手掌时，即时绘制手掌的节点。

import cv2
import numpy as np

import mediapipe as mp
from mediapipe import solutions
from mediapipe.framework.formats import landmark_pb2

BaseOptions = mp.tasks.BaseOptions
HandLandmarker = mp.tasks.vision.HandLandmarker
HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
VisionRunningMode = mp.tasks.vision.RunningMode

# 手掌偵測設定
options = HandLandmarkerOptions(
    num_hands=2,
    base_options=BaseOptions(model_asset_path='model/hand_landmarker.task'),
    running_mode=VisionRunningMode.IMAGE)

# 標記文字
MARGIN = 10  # pixels
FONT_SIZE = 1
FONT_THICKNESS = 1
HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green

# 繪製手掌骨架
def draw_landmarks_on_image(rgb_image, detection_result):
  hand_landmarks_list = detection_result.hand_landmarks
  handedness_list = detection_result.handedness
  annotated_image = np.copy(rgb_image)

  # Loop through the detected hands to visualize.
  for idx in range(len(hand_landmarks_list)):
    hand_landmarks = hand_landmarks_list[idx]
    handedness = handedness_list[idx]

    # Draw the hand landmarks.
    hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
    hand_landmarks_proto.landmark.extend([
      landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
    ])
    solutions.drawing_utils.draw_landmarks(
      annotated_image,
      hand_landmarks_proto,
      solutions.hands.HAND_CONNECTIONS,
      solutions.drawing_styles.get_default_hand_landmarks_style(),
      solutions.drawing_styles.get_default_hand_connections_style())

    # Get the top left corner of the detected hand's bounding box.
    height, width, _ = annotated_image.shape
    x_coordinates = [landmark.x for landmark in hand_landmarks]
    y_coordinates = [landmark.y for landmark in hand_landmarks]
    text_x = int(min(x_coordinates) * width)
    text_y = int(min(y_coordinates) * height) - MARGIN

    # Draw handedness (left or right hand) on the image.
    cv2.putText(annotated_image, f"{handedness[0].category_name}",
                (text_x, text_y), cv2.FONT_HERSHEY_DUPLEX,
                FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)

  return annotated_image

with HandLandmarker.create_from_options(options) as landmarker:
    cap = cv2.VideoCapture(0)   
    if not cap.isOpened():
        print("Cannot open camera")
        exit()
    while True:
        ret, frame = cap.read()
        w = frame.shape[1]                  # 畫面寬度
        h = frame.shape[0]                  # 畫面高度
        if not ret:
            print("Cannot receive frame")
            break

        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
        hand_landmarker_result = landmarker.detect(mp_image)
        
        print(hand_landmarker_result.handedness)

        annotated_image = draw_landmarks_on_image(frame, hand_landmarker_result)
        

        cv2.imshow('oxxostudio', annotated_image)
        if cv2.waitKey(5) == ord('q'):
            break    # 按下 q 鍵停止
cap.release()
cv2.destroyAllWindows()

参考数据

• Hand landmarks detection guide for Python
• Hand landmarks detection guide

意见反馈