Fingerspelling embedding + ClearML

preprocessing/extract_mediapipe_landmarks.py

@@ -35,7 +35,11 @@ class LandmarksResults:
     ):
         self.results = results
         self.num_landmarks_pose = num_landmarks_pose
-        self.num_landmarks_hand = num_landmarks_hand
+        self.num_landmarks_hand = num_landmarks_hand            
+
+    @property
+    def empty(self):
+        return self.results.pose_landmarks is None or (self.results.left_hand_landmarks is None and self.results.right_hand_landmarks is None)
 
     @property
     def pose_landmarks(self):
@@ -67,6 +71,10 @@ def get_landmarks(image_orig, holistic, debug=False):
     # Convert the BGR image to RGB before processing.
     image = cv2.cvtColor(image_orig, cv2.COLOR_BGR2RGB)
     results = LandmarksResults(holistic.process(image))
+
+    if results.empty:
+        return None
+
     if debug:
         lmks_pose = []
         for lmk in results.pose_landmarks:
@@ -94,6 +102,7 @@ def get_landmarks(image_orig, holistic, debug=False):
             len(lmks_right_hand) == 2 * LEN_LANDMARKS_HAND
         ), f"{len(lmks_right_hand)} != {2 * LEN_LANDMARKS_HAND}"
     landmarks = []
+
     for lmk in chain(
         results.pose_landmarks,
         results.left_hand_landmarks,
@@ -128,10 +137,21 @@ def extract(args):
     videos_folder = args.videos_folder
     os.makedirs(landmarks_output, exist_ok=True)
     for fn_video in tqdm(sorted(glob.glob(op.join(videos_folder, "*mp4")))):
+
+        # check if landmarks already exist
+        if op.exists(op.join(landmarks_output, op.basename(fn_video).split(".")[0] + ".npy")):
+            continue
+
         cap = cv2.VideoCapture(fn_video)
         ret, image_orig = cap.read()
         height, width = image_orig.shape[:2]
         landmarks_video = []
+
+        # make sure fps is 20 by determining the number of frames to be skipped
+        frame_rate = int(cap.get(cv2.CAP_PROP_FPS))
+        frame_skip = (frame_rate // 10) - 1
+        
+
         with tqdm(total=int(cap.get(cv2.CAP_PROP_FRAME_COUNT))) as pbar:
             with mp_holistic.Holistic(
                 static_image_mode=False,
@@ -145,7 +165,11 @@ def extract(args):
                         print(e)
                         landmarks = get_landmarks(image_orig, holistic, debug=True)
                     ret, image_orig = cap.read()
-                    landmarks_video.append(landmarks)
+                    for _ in range(frame_skip):
+                        ret, image_orig = cap.read()
+                        pbar.update(1)
+                    if landmarks:
+                        landmarks_video.append(landmarks)
                     pbar.update(1)
         landmarks_video = np.vstack(landmarks_video)
         np.save(