JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
PyHandDetector.py
Go to the documentation of this file.
1import pyjevois
2if pyjevois.pro: import libjevoispro as jevois
3else: import libjevois as jevois
4import cv2
5import numpy as np
6import mediapipe as mp
7
8## Hand detection using MediaPipe
9#
10# Detect hands using MediaPipe in Python
11#
12# This code is derived from sample_hand.py at https://github.com/Kazuhito00/mediapipe-python-sample
13#
14# @author Laurent Itti
15#
16# @videomapping JVUI 0 0 30.0 CropScale=RGB24@512x288:YUYV 1920 1080 30.0 JeVois PyHandDetector
17# @email itti\@usc.edu
18# @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
19# @copyright Copyright (C) 2021 by Laurent Itti, iLab and the University of Southern California
20# @mainurl http://jevois.org
21# @supporturl http://jevois.org/doc
22# @otherurl http://iLab.usc.edu
23# @license GPL v3
24# @distribution Unrestricted
25# @restrictions None
26# @ingroup modules
27class PyHandDetector:
28 # ###################################################################################################
29 ## Constructor
30 def __init__(self):
31 # Instantiate a JeVois Timer to measure our processing framerate:
32 self.timer = jevois.Timer("hand", 100, jevois.LOG_INFO)
33
34 # Instantiate mediapipe hand detector:
35 self.mp_hands = mp.solutions.hands
36 self.hands = self.mp_hands.Hands(max_num_hands = 2, min_detection_confidence = 0.7,
37 min_tracking_confidence = 0.5)
38 self.use_brect = True; # true to show a bounding rectangle around each hand
39
40 # ###################################################################################################
41 ## Process function with GUI output
42 def processGUI(self, inframe, helper):
43 # Start a new display frame, gets its size and also whether mouse/keyboard are idle:
44 idle, winw, winh = helper.startFrame()
45
46 # Draw full-resolution input frame from camera:
47 x, y, w, h = helper.drawInputFrame("c", inframe, False, False)
48 helper.itext('JeVois-Pro Hand Landmarks Detection')
49
50 # Get the next camera image at processing resolution (may block until it is captured):
51 image = inframe.getCvRGBp()
52 iw, ih = image.shape[1], image.shape[0]
53
54 # Start measuring image processing time:
55 self.timer.start()
56
57 # Detect hands:
58 results = self.hands.process(image)
59
60 # Draw results:
61 if results.multi_hand_landmarks is not None:
62 for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness):
63 cx, cy = self.calc_palm_moment(iw, ih, hand_landmarks)
64 self.draw_landmarks(helper, iw, ih, cx, cy, hand_landmarks, handedness)
65 if self.use_brect:
66 brect = self.calc_bounding_rect(iw, ih, hand_landmarks)
67 helper.drawRect(brect[0], brect[1], brect[2], brect[3], 0x6040ffff, True)
68
69 # Write frames/s info from our timer:
70 fps = self.timer.stop()
71 helper.iinfo(inframe, fps, winw, winh);
72
73 # End of frame:
74 helper.endFrame()
75
76 # ###################################################################################################
77 def calc_palm_moment(self, iw, ih, landmarks):
78 palm_array = np.empty((0, 2), int)
79
80 for index, landmark in enumerate(landmarks.landmark):
81 landmark_x = min(int(landmark.x * iw), iw - 1)
82 landmark_y = min(int(landmark.y * ih), ih - 1)
83
84 landmark_point = [np.array((landmark_x, landmark_y))]
85
86 if index == 0:
87 palm_array = np.append(palm_array, landmark_point, axis=0)
88 if index == 1:
89 palm_array = np.append(palm_array, landmark_point, axis=0)
90 if index == 5:
91 palm_array = np.append(palm_array, landmark_point, axis=0)
92 if index == 9:
93 palm_array = np.append(palm_array, landmark_point, axis=0)
94 if index == 13:
95 palm_array = np.append(palm_array, landmark_point, axis=0)
96 if index == 17:
97 palm_array = np.append(palm_array, landmark_point, axis=0)
98
99 M = cv2.moments(palm_array)
100 cx, cy = 0, 0
101 if M['m00'] != 0:
102 cx = int(M['m10'] / M['m00'])
103 cy = int(M['m01'] / M['m00'])
104 return cx, cy
105
106 # ###################################################################################################
107 def calc_bounding_rect(self, iw, ih, landmarks):
108 landmark_array = np.empty((0, 2), int)
109
110 for _, landmark in enumerate(landmarks.landmark):
111 landmark_x = min(int(landmark.x * iw), iw - 1)
112 landmark_y = min(int(landmark.y * ih), ih - 1)
113 landmark_point = [np.array((landmark_x, landmark_y))]
114 landmark_array = np.append(landmark_array, landmark_point, axis=0)
115 x, y, ww, hh = cv2.boundingRect(landmark_array)
116 return [x, y, x + ww, y + hh]
117
118 # ###################################################################################################
119 def draw_landmarks(self, helper, iw, ih, cx, cy, landmarks, handedness):
120 lpx = []
121 lpy = []
122 col = 0xff00ff00
123
124 for index, landmark in enumerate(landmarks.landmark):
125 if landmark.visibility < 0 or landmark.presence < 0: continue
126
127 landmark_x = min(int(landmark.x * iw), iw - 1)
128 landmark_y = min(int(landmark.y * ih), ih - 1)
129 # landmark_z = landmark.z
130
131 lpx.append(landmark_x)
132 lpy.append(landmark_y)
133
134 if index == 0:
135 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
136 if index == 1:
137 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
138 if index == 2:
139 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
140 if index == 3:
141 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
142 if index == 4:
143 helper.drawCircle(landmark_x, landmark_y, 5, col, False)
144 helper.drawCircle(landmark_x, landmark_y, 12, col, True)
145 if index == 5:
146 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
147 if index == 6:
148 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
149 if index == 7:
150 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
151 if index == 8:
152 helper.drawCircle(landmark_x, landmark_y, 5, col, False)
153 helper.drawCircle(landmark_x, landmark_y, 12, col, True)
154 if index == 9:
155 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
156 if index == 10:
157 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
158 if index == 11:
159 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
160 if index == 12:
161 helper.drawCircle(landmark_x, landmark_y, 5, col, False)
162 helper.drawCircle(landmark_x, landmark_y, 12, col, True)
163 if index == 13:
164 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
165 if index == 14:
166 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
167 if index == 15:
168 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
169 if index == 16:
170 helper.drawCircle(landmark_x, landmark_y, 5, col, False)
171 helper.drawCircle(landmark_x, landmark_y, 12, col, True)
172 if index == 17:
173 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
174 if index == 18:
175 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
176 if index == 19:
177 helper.drawCircle(landmark_x, landmark_y, 5, col, True)
178 if index == 20:
179 helper.drawCircle(landmark_x, landmark_y, 5, col, False)
180 helper.drawCircle(landmark_x, landmark_y, 12, col, True)
181
182 if len(lpx) > 0:
183 helper.drawLine(lpx[2], lpy[2], lpx[3], lpy[3], col)
184 helper.drawLine(lpx[3], lpy[3], lpx[4], lpy[4], col)
185
186 helper.drawLine(lpx[5], lpy[5], lpx[6], lpy[6], col)
187 helper.drawLine(lpx[6], lpy[6], lpx[7], lpy[7], col)
188 helper.drawLine(lpx[7], lpy[7], lpx[8], lpy[8], col)
189
190 helper.drawLine(lpx[9], lpy[9], lpx[10], lpy[10], col)
191 helper.drawLine(lpx[10], lpy[10], lpx[11], lpy[11], col)
192 helper.drawLine(lpx[11], lpy[11], lpx[12], lpy[12], col)
193
194 helper.drawLine(lpx[13], lpy[13], lpx[14], lpy[14], col)
195 helper.drawLine(lpx[14], lpy[14], lpx[15], lpy[15], col)
196 helper.drawLine(lpx[15], lpy[15], lpx[16], lpy[16], col)
197
198 helper.drawLine(lpx[17], lpy[17], lpx[18], lpy[18], col)
199 helper.drawLine(lpx[18], lpy[18], lpx[19], lpy[19], col)
200 helper.drawLine(lpx[19], lpy[19], lpx[20], lpy[20], col)
201
202 helper.drawLine(lpx[0], lpy[0], lpx[1], lpy[1], col)
203 helper.drawLine(lpx[1], lpy[1], lpx[2], lpy[2], col)
204 helper.drawLine(lpx[2], lpy[2], lpx[5], lpy[5], col)
205 helper.drawLine(lpx[5], lpy[5], lpx[9], lpy[9], col)
206 helper.drawLine(lpx[9], lpy[9], lpx[13], lpy[13], col)
207 helper.drawLine(lpx[13], lpy[13], lpx[17], lpy[17], col)
208 helper.drawLine(lpx[17], lpy[17], lpx[0], lpy[0], col)
209
210 helper.drawCircle(cx, cy, 5, 0xffff8080, True)
211
212 # If camera view was not flipped horizontally, swap left/right hands:
213 if handedness.classification[0].label[0] == 'L': hnd = 'R'
214 else: hnd = 'L'
215 helper.drawText(cx - 2, cy - 3, hnd, 0xffff0000)
PyHandDetector.PyHandDetector
Hand detection using MediaPipe.
Definition PyHandDetector.py:27
PyHandDetector.PyHandDetector.processGUI
processGUI(self, inframe, helper)
Process function with GUI output.
Definition PyHandDetector.py:42
PyHandDetector.PyHandDetector.calc_bounding_rect
calc_bounding_rect(self, iw, ih, landmarks)
Definition PyHandDetector.py:107
PyHandDetector.PyHandDetector.draw_landmarks
draw_landmarks(self, helper, iw, ih, cx, cy, landmarks, handedness)
Definition PyHandDetector.py:119
PyHandDetector.PyHandDetector.calc_palm_moment
calc_palm_moment(self, iw, ih, landmarks)
Definition PyHandDetector.py:77