How to overlay custom shapes on detected landmarks in Python using MediaPipe?

I'm working on a project using MediaPipe to detect hand landmarks. While the detection is working fine, I want to customize the visualization. Instead of the default connections and dots provided by mp_drawing.draw_landmarks(), I want to overlay custom shapes (like circles, squares, or even images) on specific landmarks.

Here’s what I’ve tried so far:

1. I used mp_drawing.draw_landmarks() to visualize the landmarks.
2. I modified the mp_drawing.DrawingSpec to change colors and thicknesses, but that still uses the default rendering.

Here’s a minimal code snippet of what I currently have:

import cv2
import mediapipe as mp

mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils

cap = cv2.VideoCapture(0)

with mp_hands.Hands() as hands:
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break

        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        results = hands.process(frame)

        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
        
        cv2.imshow("MediaPipe Hands", frame)
        if cv2.waitKey(10) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()

My Goal:

• Instead of just drawing dots, I want to add custom shapes (e.g., larger circles, squares, or images like a star icon) to specific landmarks like the wrist or fingertips.

Questions:

1. How can I overlay custom shapes or images on specific landmarks detected by MediaPipe?
2. Can I completely skip mp_drawing.draw_landmarks() and manually draw all landmarks and connections? If yes, what’s the best way to do this?

Any help or guidance would be appreciated!

Here is an example of how you can do that

import cv2
import mediapipe as mp
import numpy as np

mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils

def overlay_shape(image, landmark, shape_type='circle', color=(0, 0, 255), radius=5):
    height, width, _ = image.shape
    x = int(landmark.x * width)
    y = int(landmark.y * height)

    if shape_type == 'circle':
        cv2.circle(image, (x, y), radius, color, -1)
    elif shape_type == 'square':
        cv2.rectangle(image, (x - radius, y - radius), (x + radius, y + radius), color, -1)
    elif shape_type == 'star':
        pts = []
        for i in range(5):
            angle = i * 2 * np.pi / 5
            x_star = x + radius * np.cos(angle)
            y_star = y + radius * np.sin(angle)
            pts.append((int(x_star), int(y_star)))
        pts.append(pts[0])  # Close the polygon
        cv2.fillPoly(image, [np.array(pts)], color)

    return image

cap = cv2.VideoCapture(0)
with mp_hands.Hands() as hands:
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            print("Ignoring empty camera frame.")
            continue

    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(frame)

    frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            wrist = hand_landmarks.landmark[0]
            index_tip = hand_landmarks.landmark[8]
            middle_tip = hand_landmarks.landmark[12]
            # Apply shapes to the landmarks
            frame = overlay_shape(frame, wrist, shape_type='square', color=(0, 255, 0), radius=10)
            frame = overlay_shape(frame, index_tip, shape_type='star', color=(255, 0, 0), radius=8)
            frame = overlay_shape(frame, middle_tip, shape_type='circle', color=(0, 0, 255), radius=15)

            mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)

    cv2.imshow('MediaPipe Hands', frame)
    if cv2.waitKey(10) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

Hand landmarks index I found in Hand Tracking with 21 landmarks