{ "cells": [ { "cell_type": "code", "execution_count": 5, "id": "start", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Python: /Applications/Blender.app/Contents/Resources/5.1/python/bin/python3.13\n", "anywidget: 0.11.0\n", "ipywidgets: 8.1.8\n" ] } ], "source": [ "import anywidget\n", "\n", "import ipywidgets\n", "\n", "import sys\n", "\n", "print(\"Python:\", sys.executable)\n", "\n", "print(\"anywidget:\", anywidget.__version__)\n", "\n", "print(\"ipywidgets:\", ipywidgets.__version__)" ] }, { "cell_type": "code", "execution_count": null, "id": "dbd9bdd7-0c9c-4b51-9f7c-1cdddbdbc1d4", "metadata": {}, "outputs": [], "source": "# %% 1) Make the gesture-recognizer-widget package importable in this kernel.\n# It (and its deps) must live in BLENDER's bundled Python, which is the\n# kernel running this notebook -- sys.executable points at it, so we\n# install into it once. To hack on a local checkout instead, swap the\n# package name below for \"-e /path/to/caputre_motion\" (see README).\nimport subprocess\nimport sys\n\ntry:\n import gesture_widget # noqa: F401\nexcept ImportError:\n subprocess.check_call(\n [sys.executable, \"-m\", \"pip\", \"install\", \"gesture-recognizer-widget\"]\n )\n\nfrom gesture_widget import GestureRecognizerWidget # noqa: E402\n\nprint(\"gesture_widget ready in Blender's Python\")\n\n\n# %% 2) Show the widget, then click \"Start Camera\" in the output and allow webcam\nw = GestureRecognizerWidget()\nw.sync_interval_ms = 33 # push landmarks to Python at ~30 Hz\nw\n" }, { "cell_type": "code", "execution_count": 7, "id": "2d10158b-b23e-4e8f-a6ac-b952426a3fec", "metadata": {}, "outputs": [], "source": [ "\n", "# %% 3) A live hand armature, driven by bpy (main-thread safe).\n", "# One bone per joint connection; the timer sets each bone's head/tail\n", "# to the MediaPipe landmark positions to form a hand skeleton.\n", "import bpy # noqa: E402\n", "\n", "SCALE = 5.0 # world units spanned by the camera frame\n", "BONE_REST = 0.4 # placeholder length for the bones at creation time;\n", " # the timer overwrites head/tail with real positions.\n", "SMOOTHING = 0.4 # EMA factor for de-jitter: 1.0 = raw (no smoothing),\n", " # lower = smoother but laggier. ~0.3-0.5 is a good range.\n", "\n", "# Traitlet names on `w` for all 21 landmarks of the MediaPipe hand-landmark\n", "# model bundle. (Mapped to friendly object-name hints, though the bones get\n", "# their positions straight from the traitlets.)\n", "LANDMARKS = {\n", " \"wrist\": \"HandLM_Wrist\",\n", " \"thumb_cmc\": \"HandLM_Thumb_CMC\",\n", " \"thumb_mcp\": \"HandLM_Thumb_MCP\",\n", " \"thumb_ip\": \"HandLM_Thumb_IP\",\n", " \"thumb_tip\": \"HandLM_Thumb_Tip\",\n", " \"index_finger_mcp\": \"HandLM_Index_MCP\",\n", " \"index_finger_pip\": \"HandLM_Index_PIP\",\n", " \"index_finger_dip\": \"HandLM_Index_DIP\",\n", " \"index_finger_tip\": \"HandLM_Index_Tip\",\n", " \"middle_finger_mcp\": \"HandLM_Middle_MCP\",\n", " \"middle_finger_pip\": \"HandLM_Middle_PIP\",\n", " \"middle_finger_dip\": \"HandLM_Middle_DIP\",\n", " \"middle_finger_tip\": \"HandLM_Middle_Tip\",\n", " \"ring_finger_mcp\": \"HandLM_Ring_MCP\",\n", " \"ring_finger_pip\": \"HandLM_Ring_PIP\",\n", " \"ring_finger_dip\": \"HandLM_Ring_DIP\",\n", " \"ring_finger_tip\": \"HandLM_Ring_Tip\",\n", " \"pinky_mcp\": \"HandLM_Pinky_MCP\",\n", " \"pinky_pip\": \"HandLM_Pinky_PIP\",\n", " \"pinky_dip\": \"HandLM_Pinky_DIP\",\n", " \"pinky_tip\": \"HandLM_Pinky_Tip\",\n", "}\n", "\n", "# (parent, child) edges of the MediaPipe hand skeleton. Each edge is one bone:\n", "# head at the parent landmark, tail at the child landmark.\n", "CONNECTIONS = [\n", " (\"wrist\", \"thumb_cmc\"),\n", " (\"thumb_cmc\", \"thumb_mcp\"),\n", " (\"thumb_mcp\", \"thumb_ip\"),\n", " (\"thumb_ip\", \"thumb_tip\"),\n", " (\"wrist\", \"index_finger_mcp\"),\n", " (\"index_finger_mcp\", \"index_finger_pip\"),\n", " (\"index_finger_pip\", \"index_finger_dip\"),\n", " (\"index_finger_dip\", \"index_finger_tip\"),\n", " (\"wrist\", \"middle_finger_mcp\"),\n", " (\"middle_finger_mcp\", \"middle_finger_pip\"),\n", " (\"middle_finger_pip\", \"middle_finger_dip\"),\n", " (\"middle_finger_dip\", \"middle_finger_tip\"),\n", " (\"wrist\", \"ring_finger_mcp\"),\n", " (\"ring_finger_mcp\", \"ring_finger_pip\"),\n", " (\"ring_finger_pip\", \"ring_finger_dip\"),\n", " (\"ring_finger_dip\", \"ring_finger_tip\"),\n", " (\"wrist\", \"pinky_mcp\"),\n", " (\"pinky_mcp\", \"pinky_pip\"),\n", " (\"pinky_pip\", \"pinky_dip\"),\n", " (\"pinky_dip\", \"pinky_tip\"),\n", "]\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "cc5a6413-921a-46de-943c-1576ea829497", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Timer running — 20-bone hand armature now tracks your hand.\n" ] } ], "source": [ "from mathutils import Vector # noqa: E402\n", "\n", "# Armature object holding one bone per connection.\n", "ARM_NAME = \"HandArmature\"\n", "arm_obj = bpy.data.objects.get(ARM_NAME)\n", "if arm_obj is None:\n", " arm_obj = bpy.data.objects.new(ARM_NAME, bpy.data.armatures.new(ARM_NAME))\n", " bpy.context.scene.collection.objects.link(arm_obj)\n", "arm_obj.show_in_front = False # shaded octahedral look, like the rig\n", "arm_obj.data.display_type = \"OCTAHEDRAL\"\n", "\n", "\n", "def _bone_name(child_trait):\n", " # Each child landmark is the tail of exactly one bone, so it names it.\n", " return f\"Bone_{child_trait}\"\n", "\n", "\n", "# The bone whose TAIL sits on landmark X is the parent of the bone whose HEAD\n", "# sits on X -> this turns the flat edge list into proper finger chains.\n", "_bone_ending_at = {child: _bone_name(child) for _p, child in CONNECTIONS}\n", "\n", "\n", "# Create the bones once + parent them into finger chains. The timer drives the\n", "# real head/tail each frame; Blender draws each as a proper octahedral bone\n", "# whose width is auto-proportional to its length (no scaling involved).\n", "bpy.context.view_layer.objects.active = arm_obj\n", "bpy.ops.object.mode_set(mode=\"EDIT\")\n", "eb = arm_obj.data.edit_bones\n", "for i, (_parent, child) in enumerate(CONNECTIONS):\n", " bone = eb.get(_bone_name(child)) or eb.new(_bone_name(child))\n", " bone.head = (i * BONE_REST, 0.0, 0.0)\n", " bone.tail = (i * BONE_REST, 0.0, BONE_REST)\n", "for parent_lm, child in CONNECTIONS:\n", " bone = eb[_bone_name(child)]\n", " parent_name = _bone_ending_at.get(parent_lm) # None for the wrist roots\n", " bone.parent = eb[parent_name] if parent_name else None\n", " bone.use_connect = False # heads are driven explicitly each frame\n", "bpy.ops.object.mode_set(mode=\"OBJECT\")\n", "\n", "\n", "def _img_to_world(p):\n", " # image space (x right, y DOWN) -> Blender (x right, z UP, y depth)\n", " x, y, z = p\n", " return Vector(((x - 0.5) * SCALE, z * SCALE, (0.5 - y) * SCALE))\n", "\n", "\n", "# Per-landmark smoothing state (exponential moving average) to kill jitter.\n", "_smoothed = {}\n", "\n", "\n", "def _update_landmarks():\n", " # Read every landmark; bail until the whole hand is visible.\n", " pos = {}\n", " for trait in LANDMARKS:\n", " p = getattr(w, trait) # [x, y, z] normalized 0..1, or [] when no hand\n", " if not p:\n", " return 0.033\n", " raw = _img_to_world(p)\n", " prev = _smoothed.get(trait)\n", " # EMA: ease the stored position toward the raw sample by SMOOTHING.\n", " pos[trait] = raw if prev is None else prev.lerp(raw, SMOOTHING)\n", " _smoothed[trait] = pos[trait]\n", "\n", " # Timers run on the main thread, so entering Edit mode and writing edit\n", " # bones here is safe. Setting head/tail directly = proper bones, no scaling.\n", " try:\n", " bpy.context.view_layer.objects.active = arm_obj\n", " bpy.ops.object.mode_set(mode=\"EDIT\")\n", " eb = arm_obj.data.edit_bones\n", " for parent_lm, child in CONNECTIONS:\n", " head, tail = pos[parent_lm], pos[child]\n", " if (tail - head).length < 1e-4: # avoid zero-length bones\n", " tail = head + Vector((0.0, 0.0, 1e-3))\n", " bone = eb[_bone_name(child)]\n", " bone.head = head\n", " bone.tail = tail\n", " bpy.ops.object.mode_set(mode=\"OBJECT\")\n", " except RuntimeError:\n", " pass # context not ready this tick; retry next frame\n", " return 0.033 # seconds until next call (~30 Hz). Return None to stop.\n", "\n", "\n", "if bpy.app.timers.is_registered(_update_landmarks):\n", " bpy.app.timers.unregister(_update_landmarks)\n", "bpy.app.timers.register(_update_landmarks)\n", "\n", "print(f\"Timer running — {len(CONNECTIONS)}-bone hand armature now tracks your hand.\")\n", "\n", "\n", "# %% 4) Stop driving Blender\n", "# bpy.app.timers.unregister(_update_landmarks)\n", "# w.stop() # turn the camera off\n" ] }, { "cell_type": "code", "execution_count": null, "id": "fc266a95-7bf4-4aa1-a47b-04e202e5db67", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.9" } }, "nbformat": 4, "nbformat_minor": 5 }