Hand_Tracking.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "806935f6",
   "metadata": {},
   "source": [
    "### Hand Tracking\n",
    "\n",
    "MediaPipe and OpenAI Gym for a cart pole that is controlled from hand position in the webcam. See [Hand Tracking](https://apmonitor.com/pds/index.php/Main/HandTracking) in [Machine Learning for Engineers](https://apmonitor.com/pds).\n",
    "\n",
    "<img width=500px align=left src='https://apmonitor.com/pds/uploads/Main/hand_tracking.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00597a9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install opencv-python mediapipe pyglet pygame gymnasium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a975b94",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://google.github.io/mediapipe/solutions/hands\n",
    "import cv2\n",
    "import mediapipe as mp\n",
    "import os\n",
    "import urllib.request\n",
    "\n",
    "mp_drawing = mp.solutions.drawing_utils\n",
    "mp_drawing_styles = mp.solutions.drawing_styles\n",
    "mp_hands = mp.solutions.hands\n",
    "\n",
    "# download image as hands.jpg\n",
    "url = 'http://apmonitor.com/pds/uploads/Main/hands.jpg'\n",
    "urllib.request.urlretrieve(url, 'hands.jpg')\n",
    "IMAGE_FILES = ['hands.jpg']\n",
    "with mp_hands.Hands(\n",
    "    static_image_mode=True,\n",
    "    max_num_hands=2,\n",
    "    min_detection_confidence=0.5) as hands:\n",
    "  for idx, file in enumerate(IMAGE_FILES):\n",
    "    image = cv2.flip(cv2.imread(file), 1)\n",
    "\n",
    "    # Convert the BGR image to RGB before processing.\n",
    "    results = hands.process(cv2.cvtColor(image, \\\n",
    "                            cv2.COLOR_BGR2RGB))\n",
    "\n",
    "    # Print handedness and draw hand landmarks on the image.\n",
    "    print('Handedness:', results.multi_handedness)\n",
    "    if not results.multi_hand_landmarks:\n",
    "      continue\n",
    "    image_height, image_width, _ = image.shape\n",
    "    annotated_image = image.copy()\n",
    "    for hand_landmarks in results.multi_hand_landmarks:\n",
    "      tip = mp_hands.HandLandmark.INDEX_FINGER_TIP\n",
    "      print('hand_landmarks:', hand_landmarks)\n",
    "      print(\n",
    "          f'Index finger tip coordinates: (',\n",
    "          f'{hand_landmarks.landmark[tip].x * image_width}, '\n",
    "          f'{hand_landmarks.landmark[tip].y * image_height})'\n",
    "      )\n",
    "      mp_drawing.draw_landmarks(\n",
    "          annotated_image,\n",
    "          hand_landmarks,\n",
    "          mp_hands.HAND_CONNECTIONS,\n",
    "          mp_drawing_styles.get_default_hand_landmarks_style(),\n",
    "          mp_drawing_styles.get_default_hand_connections_style())\n",
    "    cv2.imwrite('annotated_image' + str(idx) + '.png', \\\n",
    "                cv2.flip(annotated_image, 1))\n",
    "    # Draw hand world landmarks.\n",
    "    if not results.multi_hand_world_landmarks:\n",
    "      continue\n",
    "    for hand_world_landmarks in results.multi_hand_world_landmarks:\n",
    "      mp_drawing.plot_landmarks(\n",
    "        hand_world_landmarks, \\\n",
    "        mp_hands.HAND_CONNECTIONS, azimuth=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8cca3bb",
   "metadata": {},
   "source": [
    "<img width=400px align=left src='http://apmonitor.com/pds/uploads/Main/cartpole.gif'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5114054",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "env = gym.make(\"CartPole-v1\",render_mode='human')\n",
    "observation = env.reset()\n",
    "for _ in range(100):\n",
    "  env.render()\n",
    "  # Input:\n",
    "  #   Force to the cart with actions: 0=left, 1=right\n",
    "  # Returns:\n",
    "  #   obs = cart position, cart velocity, pole angle, rot rate\n",
    "  #   reward = +1 for every timestep\n",
    "  #   done = True when abs(angle)>15 or abs(cart pos)>2.4\n",
    "  action = env.action_space.sample() # random action\n",
    "  observation, reward, done, info, e = env.step(action)\n",
    "\n",
    "  if done:\n",
    "    observation = env.reset()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b97e83",
   "metadata": {},
   "source": [
    "<img align=left width=500px src='https://apmonitor.com/pds/uploads/Main/hand_cart.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1b6214",
   "metadata": {},
   "outputs": [],
   "source": [
    "# may not work in Jupyter Notebook\n",
    "# run as a script from the terminal\n",
    "import cv2\n",
    "import mediapipe as mp\n",
    "import os\n",
    "import gymnasium as gym\n",
    "import random\n",
    "import warnings\n",
    "import time\n",
    "\n",
    "env = gym.make(\"CartPole-v1\",render_mode='rgb_array')\n",
    "observation = env.reset()\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "\n",
    "mp_drawing = mp.solutions.drawing_utils\n",
    "mp_drawing_styles = mp.solutions.drawing_styles\n",
    "mp_hands = mp.solutions.hands\n",
    "\n",
    "# Webcam input\n",
    "error_count = 0\n",
    "cap = cv2.VideoCapture(0)\n",
    "with mp_hands.Hands(model_complexity=0,min_detection_confidence=0.5,\n",
    "                    min_tracking_confidence=0.5) as hands:\n",
    "  while cap.isOpened():\n",
    "    success, image = cap.read()\n",
    "    if image is None:\n",
    "      time.sleep(0.1)\n",
    "      error_count += 1\n",
    "      if error_count<100:\n",
    "        continue\n",
    "      else:\n",
    "        print('Error reading from webcam')\n",
    "        break\n",
    "    else:\n",
    "      error_count = 0\n",
    "    image.flags.writeable = False\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    results = hands.process(image)\n",
    "\n",
    "    # Draw the hand annotations on the image.\n",
    "    image.flags.writeable = True\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
    "    hand_landmarks = None\n",
    "    if results.multi_hand_landmarks:\n",
    "      for hand_landmarks in results.multi_hand_landmarks:\n",
    "        mp_drawing.draw_landmarks(\n",
    "            image,\n",
    "            hand_landmarks,\n",
    "            mp_hands.HAND_CONNECTIONS,\n",
    "            mp_drawing_styles.get_default_hand_landmarks_style(),\n",
    "            mp_drawing_styles.get_default_hand_connections_style())\n",
    "    # Flip the image horizontally for a selfie-view display.\n",
    "    image2 = cv2.flip(image, 1)\n",
    "    # Resize image\n",
    "    width = 800\n",
    "    height = int(image.shape[0]*(width/image2.shape[1]))\n",
    "    dim = (width,height)\n",
    "    image2 = cv2.resize(image2, dim)\n",
    "\n",
    "    if hand_landmarks:\n",
    "      h, w, _ = image.shape\n",
    "      t = mp_hands.HandLandmark.INDEX_FINGER_TIP\n",
    "      x = hand_landmarks.landmark[t].x\n",
    "      y = hand_landmarks.landmark[t].y\n",
    "    else:\n",
    "      x = 0.5\n",
    "\n",
    "    cart = env.render()\n",
    "    # take bottom 2/3 of image\n",
    "    cart = cart[int(cart.shape[0]/3):]\n",
    "    height = int(cart.shape[0] * (image2.shape[1]/cart.shape[1]))\n",
    "    dim = (image2.shape[1], height)\n",
    "    cart = cv2.resize(cart, dim)\n",
    "    image3 = cv2.vconcat([cart,image2])\n",
    "    cv2.imshow('Hands - CartPole', image3)\n",
    "    if cv2.waitKey(5) & 0xFF == 27:\n",
    "      break\n",
    "        \n",
    "    r = random.random()\n",
    "    if x<0.3:\n",
    "      action = 1\n",
    "    elif x>0.7:\n",
    "      action = 0\n",
    "    else:\n",
    "      if r>=x:\n",
    "        action = 1\n",
    "      else:\n",
    "        action = 0\n",
    "    observation, reward, done, info, e = env.step(action)\n",
    "\n",
    "    if abs(observation[0])>3:\n",
    "      observation = env.reset()\n",
    "\n",
    "env.close()    \n",
    "cap.release()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3267799",
   "metadata": {},
   "source": [
    "<img width=300px align=left src='https://apmonitor.com/pds/uploads/Main/tclab_transparent.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aaaab8e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tclab\n",
    "import time\n",
    "a = tclab.TCLab()\n",
    "# Blink LED 10 times\n",
    "for i in range(10):\n",
    "    a.LED(100)      # LED On, 100%\n",
    "    time.sleep(0.5)\n",
    "    a.LED(0)        # LED Off\n",
    "    time.sleep(0.5)  \n",
    "a.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "846bffd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import mediapipe as mp\n",
    "import os\n",
    "import random\n",
    "import tclab\n",
    "import time\n",
    "\n",
    "mp_drawing = mp.solutions.drawing_utils\n",
    "mp_drawing_styles = mp.solutions.drawing_styles\n",
    "mp_hands = mp.solutions.hands\n",
    "\n",
    "# Webcam input\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "def blink_once(lab,period,br):\n",
    "    lab.LED(br)\n",
    "    time.sleep(period)\n",
    "    lab.LED(0)\n",
    "\n",
    "with tclab.TCLab() as lab:\n",
    "    with mp_hands.Hands(\n",
    "        model_complexity=0,\n",
    "        min_detection_confidence=0.5,\n",
    "        min_tracking_confidence=0.5) as hands:\n",
    "        while cap.isOpened():\n",
    "            success, image = cap.read()\n",
    "            image.flags.writeable = False\n",
    "            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "            results = hands.process(image)\n",
    "\n",
    "            # Draw the hand annotations on the image.\n",
    "            image.flags.writeable = True\n",
    "            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
    "            hand_landmarks = None\n",
    "            if results.multi_hand_landmarks:\n",
    "                for hand_landmarks in results.multi_hand_landmarks:\n",
    "                    mp_drawing.draw_landmarks(\n",
    "                    image,\n",
    "                    hand_landmarks,\n",
    "                    mp_hands.HAND_CONNECTIONS,\n",
    "                    mp_drawing_styles.get_default_hand_landmarks_style(),\n",
    "                    mp_drawing_styles.get_default_hand_connections_style())\n",
    "            # Flip the image horizontally for a selfie-view display.\n",
    "            image2 = cv2.flip(image, 1)\n",
    "            cv2.imshow('MediaPipe Hands', image2)\n",
    "            if cv2.waitKey(5) & 0xFF == 27:\n",
    "                break\n",
    "\n",
    "            if hand_landmarks:\n",
    "                h, w, _ = image.shape\n",
    "                t = mp_hands.HandLandmark.INDEX_FINGER_TIP\n",
    "                x = hand_landmarks.landmark[t].x\n",
    "                y = hand_landmarks.landmark[t].y\n",
    "            else:\n",
    "                x = y = 0.5\n",
    "\n",
    "            # adjust blink rate and brightness\n",
    "            pause = (1-x)*0.5\n",
    "            bright = (1-y)*50        \n",
    "            print(f'Pause: {pause}, Brightness: {bright}')\n",
    "            blink_once(lab,pause, bright)\n",
    "    cap.release()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}