feat(dgs_types): add process_pose feature

Author: AmitMY

examples/load_all.py

@@ -1,131 +1,18 @@
-# import itertools
-#
-# import tensorflow_datasets as tfds
-# from dotenv import load_dotenv
-#
-# # noinspection PyUnresolvedReferences
-# # from sign_language_datasets.datasets.dgs_corpus import DgsCorpusConfig
-# from sign_language_datasets.datasets.dgs_corpus import DgsCorpusConfig
-#
-# import sign_language_datasets.datasets
-#
-# # noinspection PyUnresolvedReferences
-# # import sign_language_datasets.datasets.dgs_corpus
-# from sign_language_datasets.datasets.config import SignDatasetConfig
-# # from sign_language_datasets.datasets.dgs_corpus.dgs_corpus import DgsCorpusConfig
-#
-# load_dotenv()
-#
-# # config = SignDatasetConfig(name="only-annotations", version="3.0.0", include_video=False)
-# # rwth_phoenix2014_t = tfds.load(name="rwth_phoenix2014_t", builder_kwargs=dict(config=config))
-#
-# # config = SignDatasetConfig(name="256x256:10", include_video=True, fps=10, resolution=(256, 256))
-#
-# # aslg_pc12 = tfds.load('aslg_pc12')
-# #
-# # rwth_phoenix2014_t = tfds.load('rwth_phoenix2014_t', builder_kwargs=dict(config=config))
-#
-# # wlasl = tfds.load('wlasl', builder_kwargs=dict(config=config))
-# #
-# # autsl = tfds.load('autsl', builder_kwargs=dict(
-# #     config=SignDatasetConfig(name="test", include_video=False, include_pose="holistic"),
-# # ))
-#
-# # dgs_config = DgsCorpusConfig(name="sentence-test-video", data_type="sentence",
-# #                              include_video=False, process_video=False, include_pose=None)
-# # dgs_corpus = tfds.load('dgs_corpus', builder_kwargs=dict(config=dgs_config))
-# #
-# # for datum in itertools.islice(dgs_corpus["train"], 0, 10):
-# #   print(datum)
-#
-#
-# config = SignDatasetConfig(name="signbank-annotations", version="1.0.0", include_video=False)
-# signbank = tfds.load('sign_bank', builder_kwargs=dict(config=config))
-#
-# # config = SignDatasetConfig(name="signsuisse3", version="1.0.0", include_video=False, include_pose="holistic")
-# # signsuisse = tfds.load('sign_suisse', builder_kwargs=dict(config=config))
-#
-# # print([d["p.ose"]["data"].shape for d in iter(autsl["train"])])
-# # print([d["video"].shape for d in iter(autsl["train"])])
-#
-# # config = SignDatasetConfig(name="include4", version="1.0.0", extra={"PHPSESSID": "hj9co07ct7f5noq529no9u09l4"})
-# # signtyp = tfds.load(name='sign_typ', builder_kwargs=dict(config=config))
-# #
-# # for datum in itertools.islice(signtyp["train"], 0, 10):
-# #   print(datum['sign_writing'].numpy().decode('utf-8'), datum['video'].numpy().decode('utf-8'))
-# #
-# # config = SignDatasetConfig(name="poses_1", version="1.0.0", include_video=False, include_pose="holistic")
-# # dicta_sign = tfds.load(name='dicta_sign', builder_kwargs={"config": config})
-#
-# # config = SignDatasetConfig(name="only-annotations5", version="1.0.0", include_video=False, process_video=False, include_pose="holistic")
-# # dataset = tfds.load(name='sign_bank', builder_kwargs=dict(config=SignDatasetConfig(name="annotations")))
-# #
-# # decode_str = lambda s: s.numpy().decode('utf-8')
-# # for datum in itertools.islice(dataset["train"], 0, 10):
-# #     hamnosys = decode_str(datum['hamnosys'])
-# #     glosses = [decode_str(g) for g in datum["glosses"]]
-# #     print(hamnosys, glosses)
-#
-#
-# #
-# # import tensorflow_datasets as tfds
-# # # noinspection PyUnresolvedReferences
-# # import sign_language_datasets.datasets
-# # from sign_language_datasets.datasets.config import SignDatasetConfig
-# #
-# # # Populate your access tokens
-# # TOKENS = {
-# #     "zenodo_focusnews_token": "TODO",
-# #     "zenodo_srf_videos_token": "TODO",
-# #     "zenodo_srf_poses_token": "TODO"
-# # }
-# #
-# # # Load only the annotations, and include path to video files
-# # config = SignDatasetConfig(name="annotations", version="1.0.0", process_video=False)
-# # wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS})
-# #
-# # # Load the annotations and openpose poses
-# # config = SignDatasetConfig(name="openpose", version="1.0.0", process_video=False, include_pose='openpose')
-# # wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS})
-# #
-# # # Load the annotations and mediapipe holistic poses
-# # config = SignDatasetConfig(name="holistic", version="1.0.0", process_video=False, include_pose='holistic')
-# # wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS})
-# #
-# # # Load the full video frames as a tensor
-# # config = SignDatasetConfig(name="videos", version="1.0.0", process_video=True)
-# # wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS})
-# #
-# decode_str = lambda s: s.numpy().decode('utf-8')
-# for datum in itertools.islice(signsuisse["train"], 0, 10):
-#     print(datum)
-#     print(datum["pose"])
-#     print('\n')
-#
-#
-#
+import itertools
 
-import tensorflow_datasets as tfds
-import sign_language_datasets.datasets
 from sign_language_datasets.datasets.config import SignDatasetConfig
+import tensorflow_datasets as tfds
 
-import itertools
-#
-# config = SignDatasetConfig(name="holistic-poses", version="3.0.0", include_video=False, include_pose="holistic")
-# rwth_phoenix2014_t = tfds.load(name='rwth_phoenix2014_t', builder_kwargs=dict(config=config))
-#
-# for datum in itertools.islice(rwth_phoenix2014_t["train"], 0, 10):
-#     print(datum['gloss'].numpy().decode('utf-8'))
-#     print(datum['text'].numpy().decode('utf-8'))
-#     print(datum['pose']['data'].shape)
-#     print()
-
+config = SignDatasetConfig(
+    name="pose_holistic_paths2",
+    version="3.0.0",
+    include_video=False,
+    include_pose="holistic",
+    process_pose=False
+)
 
-config = SignDatasetConfig(name="holistic-poses", version="1.0.0", include_video=False, include_pose="holistic")
-mediapi_skel = tfds.load(name='mediapi_skel', builder_kwargs=dict(config=config))
+# Load the dgs_types dataset with the specified configuration
+dgs_types = tfds.load('dgs_types', builder_kwargs=dict(config=config))
 
-for datum in itertools.islice(mediapi_skel["test"], 0, 10):
-    print(datum['id'].numpy().decode('utf-8'))
-    print(datum['subtitles'])
-    print(datum['pose']['data'].shape)
-    print()
+for datum in dgs_types["train"].take(10):
+    print(datum)

sign_language_datasets/datasets/autsl/autsl.py

@@ -9,7 +9,7 @@
 import tensorflow as tf
 import tensorflow_datasets as tfds
 from tensorflow.io.gfile import GFile
-from tensorflow_datasets.core.download.resource import get_dl_dirname
+from tensorflow_datasets.core.download.resource import get_dl_fname
 from tqdm import tqdm
 
 from ..warning import dataset_warning
@@ -123,7 +123,7 @@ def _download_and_extract_multipart(self, dl_manager: tfds.download.DownloadMana
             print(_OPENPOSE_DISCLAIMER)
 
         # Make sure not already downloaded
-        dirname = get_dl_dirname(url)
+        dirname = get_dl_fname(url)
         output_path = os.path.join(dl_manager._download_dir, dirname)
         output_path_extracted = os.path.join(dl_manager._extract_dir, dirname)
 

sign_language_datasets/datasets/config.py

@@ -16,6 +16,7 @@ def __init__(
         include_video: bool = True,
         process_video: bool = None,
         include_pose: Optional[str] = None,
+        process_pose: bool = True,
         fps: Optional[float] = None,
         resolution: Optional[Tuple[int, int]] = None,
         sample_size: Optional[int] = None,
@@ -36,6 +37,7 @@ def __init__(
         self.include_video = include_video
         self.process_video = process_video if process_video is not None else include_video
         self.include_pose = include_pose.lower() if include_pose is not None else None
+        self.process_pose = process_pose
 
         self.fps = fps
         self.resolution = resolution

sign_language_datasets/datasets/dgs_types/dgs_types.py

@@ -61,10 +61,15 @@ def _info(self) -> tfds.core.DatasetInfo:
 
         # Add poses if requested
         if self._builder_config.include_pose == "holistic":
-            pose_header_path = _POSE_HEADERS[self._builder_config.include_pose]
-            stride = 1 if self._builder_config.fps is None else 25 / self._builder_config.fps
-            pose_shape = (None, 1, 576, 3)
-            video_feature["pose"] = PoseFeature(shape=pose_shape, stride=stride, header_path=pose_header_path)
+            if self._builder_config.process_pose:
+                pose_header_path = _POSE_HEADERS[self._builder_config.include_pose]
+                stride = 1 if self._builder_config.fps is None else 25 / self._builder_config.fps
+                pose_shape = (None, 1, 576, 3)
+                pose_feature = PoseFeature(shape=pose_shape, stride=stride, header_path=pose_header_path)
+            else:
+                pose_feature = tfds.features.Text()
+
+            video_feature["pose"] = pose_feature
 
         features = {
             "id": tfds.features.Text(),
@@ -100,6 +105,7 @@ def get_galex_data(self, dl_manager: tfds.download.DownloadManager):
                 content = f.read()
                 gloss = re.findall(r"span class=\"Gloss\">(.*?)<", content)[0]
                 video = re.findall(r"source src=\"\.\.\/(.*?)\"", content)[0]
+                video = video.replace('filmekl/', 'filmegr/') # higher resolution videos
                 datum = {
                     "id": "galex_" + gloss,
                     "glosses": [gloss],
@@ -204,9 +210,9 @@ def _generate_examples(self, data, poses):
                 if self._builder_config.include_pose:
                     pose_view = f"{datum['id']}_{view['name']}"
                     if pose_view in poses:
-                        view["pose"] = poses[pose_view]
+                        view["pose"] = str(poses[pose_view])
                     else:
                         print(f"No pose for {pose_view}.")
-                        view["pose"] = None
+                        view["pose"] = ""
 
             yield datum["id"], datum

sign_language_datasets/utils/features/pose_feature.py

@@ -152,7 +152,7 @@ def encode_body(self, file_path: str, body: PoseBody):
     def encode_example(self, pose_path_or_fobj):
         """Convert the given image into a dict convertible to tf example."""
 
-        if pose_path_or_fobj is None:
+        if pose_path_or_fobj is None or pose_path_or_fobj == "":
             # Create 0 size tensors
             data_shape = list(self._shape)
             data_shape[0] = 0