Add gesture classification for handpose estimation (#168)

Author: WanliZhong

models/handpose_estimation_mediapipe/README.md

@@ -4,6 +4,9 @@ This model estimates 21 hand keypoints per detected hand from [palm detector](..
 
 ![MediaPipe Hands Keypoints](./example_outputs/hand_keypoints.png)
 
+Hand gesture classification demo (0-9)
+![hand gestures](./example_outputs/gesture_classification.png)
+
 This model is converted from TFlite to ONNX using following tools:
 - TFLite model to ONNX: https://github.com/onnx/tensorflow-onnx
 - simplified by [onnx-simplifier](https://github.com/daquexian/onnx-simplifier)

models/handpose_estimation_mediapipe/demo.py

@@ -85,6 +85,9 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
  for p in landmarks:
  cv.circle(image, p, thickness, (0, 0, 255), -1)
 
+ # used for gesture classification
+ gc = GestureClassification()
+
  for idx, handpose in enumerate(hands):
  conf = handpose[-1]
  bbox = handpose[0:4].astype(np.int32)
@@ -96,11 +99,14 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
  landmarks_screen = handpose[4:67].reshape(21, 3).astype(np.int32)
  landmarks_word = handpose[67:130].reshape(21, 3)
 
+ gesture = gc.classify(landmarks_screen)
+
  # Print results
  if print_result:
  print('-----------hand {}-----------'.format(idx + 1))
  print('conf: {:.2f}'.format(conf))
  print('handedness: {}'.format(handedness_text))
+ print('gesture: {}'.format(gesture))
  print('hand box: {}'.format(bbox))
  print('hand landmarks: ')
  for l in landmarks_screen:
@@ -113,6 +119,8 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
  cv.rectangle(display_screen, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
  # draw handedness
  cv.putText(display_screen, '{}'.format(handedness_text), (bbox[0], bbox[1] + 12), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+ # draw gesture
+ cv.putText(display_screen, '{}'.format(gesture), (bbox[0], bbox[1] + 30), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
  # Draw line between each key points
  landmarks_xy = landmarks_screen[:, 0:2]
  draw_lines(display_screen, landmarks_xy, is_draw_point=False)
@@ -149,6 +157,118 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
 
  return display_screen, display_3d
 
+class GestureClassification:
+ def _vector_2_angle(self, v1, v2):
+ uv1 = v1 / np.linalg.norm(v1)
+ uv2 = v2 / np.linalg.norm(v2)
+ angle = np.degrees(np.arccos(np.dot(uv1, uv2)))
+ return angle
+
+ def _hand_angle(self, hand):
+ angle_list = []
+ # thumb
+ angle_ = self._vector_2_angle(
+ np.array([hand[0][0] - hand[2][0], hand[0][1] - hand[2][1]]),
+ np.array([hand[3][0] - hand[4][0], hand[3][1] - hand[4][1]])
+ )
+ angle_list.append(angle_)
+ # index
+ angle_ = self._vector_2_angle(
+ np.array([hand[0][0] - hand[6][0], hand[0][1] - hand[6][1]]),
+ np.array([hand[7][0] - hand[8][0], hand[7][1] - hand[8][1]])
+ )
+ angle_list.append(angle_)
+ # middle
+ angle_ = self._vector_2_angle(
+ np.array([hand[0][0] - hand[10][0], hand[0][1] - hand[10][1]]),
+ np.array([hand[11][0] - hand[12][0], hand[11][1] - hand[12][1]])
+ )
+ angle_list.append(angle_)
+ # ring
+ angle_ = self._vector_2_angle(
+ np.array([hand[0][0] - hand[14][0], hand[0][1] - hand[14][1]]),
+ np.array([hand[15][0] - hand[16][0], hand[15][1] - hand[16][1]])
+ )
+ angle_list.append(angle_)
+ # pink
+ angle_ = self._vector_2_angle(
+ np.array([hand[0][0] - hand[18][0], hand[0][1] - hand[18][1]]),
+ np.array([hand[19][0] - hand[20][0], hand[19][1] - hand[20][1]])
+ )
+ angle_list.append(angle_)
+ return angle_list
+
+ def _finger_status(self, lmList):
+ fingerList = []
+ originx, originy = lmList[0]
+ keypoint_list = [[5, 4], [6, 8], [10, 12], [14, 16], [18, 20]]
+ for point in keypoint_list:
+ x1, y1 = lmList[point[0]]
+ x2, y2 = lmList[point[1]]
+ if np.hypot(x2 - originx, y2 - originy) > np.hypot(x1 - originx, y1 - originy):
+ fingerList.append(True)
+ else:
+ fingerList.append(False)
+
+ return fingerList
+
+ def _classify(self, hand):
+ thr_angle = 65.
+ thr_angle_thumb = 30.
+ thr_angle_s = 49.
+ gesture_str = "Undefined"
+
+ angle_list = self._hand_angle(hand)
+
+ thumbOpen, firstOpen, secondOpen, thirdOpen, fourthOpen = self._finger_status(hand)
+ # Number
+ if (angle_list[0] > thr_angle_thumb) and (angle_list[1] > thr_angle) and (angle_list[2] > thr_angle) and (
+ angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+ not firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+ gesture_str = "Zero"
+ elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] > thr_angle) and (
+ angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+ firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+ gesture_str = "One"
+ elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+ angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+ not thumbOpen and firstOpen and secondOpen and not thirdOpen and not fourthOpen:
+ gesture_str = "Two"
+ elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+ angle_list[3] < thr_angle_s) and (angle_list[4] > thr_angle) and \
+ not thumbOpen and firstOpen and secondOpen and thirdOpen and not fourthOpen:
+ gesture_str = "Three"
+ elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+ angle_list[3] < thr_angle_s) and (angle_list[4] < thr_angle) and \
+ firstOpen and secondOpen and thirdOpen and fourthOpen:
+ gesture_str = "Four"
+ elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+ angle_list[3] < thr_angle_s) and (angle_list[4] < thr_angle_s) and \
+ thumbOpen and firstOpen and secondOpen and thirdOpen and fourthOpen:
+ gesture_str = "Five"
+ elif (angle_list[0] < thr_angle_s) and (angle_list[1] > thr_angle) and (angle_list[2] > thr_angle) and (
+ angle_list[3] > thr_angle) and (angle_list[4] < thr_angle_s) and \
+ thumbOpen and not firstOpen and not secondOpen and not thirdOpen and fourthOpen:
+ gesture_str = "Six"
+ elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] > thr_angle) and (
+ angle_list[3] > thr_angle) and (angle_list[4] > thr_angle_s) and \
+ thumbOpen and firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+ gesture_str = "Seven"
+ elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] < thr_angle) and (
+ angle_list[3] > thr_angle) and (angle_list[4] > thr_angle_s) and \
+ thumbOpen and firstOpen and secondOpen and not thirdOpen and not fourthOpen:
+ gesture_str = "Eight"
+ elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] < thr_angle) and (
+ angle_list[3] < thr_angle) and (angle_list[4] > thr_angle_s) and \
+ thumbOpen and firstOpen and secondOpen and thirdOpen and not fourthOpen:
+ gesture_str = "Nine"
+
+ return gesture_str
+
+ def classify(self, landmarks):
+ hand = landmarks[:21, :2]
+ gesture = self._classify(hand)
+ return gesture
 
 if __name__ == '__main__':
  backend_id = backend_target_pairs[args.backend_target][0]