hand-detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from commonfunctions import *\n",
    "import numpy as np\n",
    "import cv2 as cv\n",
    "import mediapipe as mp\n",
    "from time import sleep\n",
    "from functools import cmp_to_key\n",
    "import os\n",
    "import pyautogui as pg\n",
    "# import scipy.ndimage import imread\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get threshold value\n",
    "def calcThreshold(hist, accHist, iFrom, iTo):\n",
    "    iFrom, iTo = int(iFrom), int(iTo)\n",
    "    numOfPixels = accHist[iTo] - (accHist[iFrom - 1] if iFrom > 0 else 0)\n",
    "    mean = np.sum(\n",
    "        hist[iFrom:iTo+1] * np.arange(iFrom, iTo+1)\n",
    "    ) / numOfPixels\n",
    "    return round(mean)\n",
    "\n",
    "# np.arange([1 , 2] < 3)\n",
    "\n",
    "\n",
    "def avgThreshold(hist, accHist, Tinit):\n",
    "    mean1 = calcThreshold(hist, accHist, 0, Tinit - 1)\n",
    "    mean2 = calcThreshold(hist, accHist, Tinit, hist.shape[0] - 1)\n",
    "    newThreshold = round((mean1 + mean2) / 2)\n",
    "    return newThreshold if Tinit == newThreshold else avgThreshold(hist, accHist, newThreshold)\n",
    "\n",
    "\n",
    "def getGlobalThreshold(img):\n",
    "    img2 = np.copy(img)\n",
    "    hist = np.histogram(img2, bins=np.arange(256))[0]\n",
    "    accHist = np.cumsum(hist)\n",
    "    Tinit = calcThreshold(hist, accHist, 0, hist.shape[0] - 1)\n",
    "    return avgThreshold(hist, accHist, Tinit)# get threshold\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_image_with_skin_color(img):\n",
    "    R = img[:, :, 0]\n",
    "    G = img[:, :, 1]\n",
    "    B = img[:, :, 2]\n",
    "    Y = 0.299 * R + 0.587 * G + 0.114 * B\n",
    "    Cb = (B - Y) * 0.564 + 128\n",
    "    Cr = (R - Y) * 0.713 + 128\n",
    "    outImg = (Cb >= 77) * (Cb <= 127) * (Cr >= 133) * (Cr <= 187)\n",
    "    return outImg\n",
    "\n",
    "\n",
    "def get_image_with_skin_color_hsv(img):\n",
    "    lower_hsv = np.array([0, 48, 80], dtype=\"uint8\")\n",
    "    upper_hsv = np.array([20, 255, 255], dtype=\"uint8\")\n",
    "    hsv_image = cv.cvtColor(img, cv.COLOR_RGB2HSV)\n",
    "    skin_region = cv.inRange(hsv_image, lower_hsv, upper_hsv)\n",
    "    return skin_region\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def test_all_images():\n",
    "    for file_name in os.listdir('./images/'):\n",
    "        if not file_name.startswith('image'):\n",
    "            continue\n",
    "        img = cv.imread(f'./images/{file_name}')\n",
    "        img = cv.cvtColor(img, cv.COLOR_BGR2RGB)\n",
    "        test1 = get_image_with_skin_color(img)\n",
    "        test2 = get_image_with_skin_color_hsv(img)\n",
    "        # show_images([img, test1, test2], ['img', 'y', 'hsv'])\n",
    "\n",
    "\n",
    "# test_all_images()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def enhance_image(img):\n",
    "    se = np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]).astype('uint8')\n",
    "    enhanced_image = np.array(img).astype('uint8')\n",
    "    enhanced_image = cv.dilate(\n",
    "        enhanced_image, se, iterations=7\n",
    "    )\n",
    "    enhanced_image = cv.erode(\n",
    "        enhanced_image, se, iterations=4\n",
    "    )\n",
    "    return enhanced_image\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def compare(item1, item2):\n",
    "    return item1[0] - item2[0]\n",
    "\n",
    "\n",
    "def detectHand(binaryImg, mainImg):\n",
    "    contours = find_contours(binaryImg,  fully_connected='high')\n",
    "    contours = [\n",
    "        [\n",
    "            int(np.min(contour[:, 0])), int(np.max(contour[:, 0])),\n",
    "            int(np.min(contour[:, 1])), int(np.max(contour[:, 1]))\n",
    "        ]\n",
    "        for contour in contours]\n",
    "    contours = sorted(contours, key=cmp_to_key(compare))\n",
    "    # temp_image = np.zeros_like(mainImg)\n",
    "    for contour in contours:\n",
    "        [Xmin, Xmax, Ymin, Ymax] = contour\n",
    "        if Xmax - Xmin >= 50 and Ymax - Ymin >= 50:\n",
    "            temp_image = mainImg[max(Xmin-70,0):min(Xmax+70,mainImg.shape[0]), max(Ymin-70,0):min(Ymax+70,mainImg.shape[1])]\n",
    "            # show_images([temp_image,binaryImg[Xmin:Xmax, Ymin:Ymax]])\n",
    "            mp_hands = mp.solutions.hands\n",
    "            hand = mp_hands.Hands()\n",
    "            result = hand.process(temp_image)\n",
    "            if result.multi_hand_landmarks:\n",
    "                return np.array(binaryImg[Xmin:Xmax, Ymin:Ymax]),(Xmin,Xmax,Ymin,Ymax)\n",
    "    return None, (0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # test detectHand\n",
    "# img1 = cv.imread('./images/image20.jpg')\n",
    "# img2 = cv.imread('./images/image21.jpg')\n",
    "# img1 = cv.cvtColor(img1, cv.COLOR_BGR2RGB)\n",
    "# img2 = cv.cvtColor(img2, cv.COLOR_BGR2RGB)\n",
    "\n",
    "# b1 = get_image_with_skin_color(img1)\n",
    "# b2 = get_image_with_skin_color(img2)\n",
    "# show_images([img1, b1])\n",
    "# print(img1.shape)\n",
    "# handImg, _ = detectHand(b1, img1)\n",
    "# show_images([handImg])\n",
    "# print('--------------------------------')\n",
    "# detectHand(b2, img2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def captureImage(cap):\n",
    "    success, img = cap.read()\n",
    "    img = cv.flip(img, 1)\n",
    "    imgRGB = cv.cvtColor(img, cv.COLOR_BGR2RGB)\n",
    "    # print(imgRGB)\n",
    "    if cv.waitKey(1) & 0xff == ord('q'):\n",
    "        cv.destroyAllWindows()\n",
    "        cap.release()\n",
    "        exit(0)\n",
    "    return imgRGB\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compareFingers(contour1, contour2):\n",
    "    return int(np.max(contour1[:, 0])-np.min(contour1[:, 0])) - int(np.max(contour2[:, 0])-np.min(contour2[:, 0]))\n",
    "\n",
    "\n",
    "def detectFingers(original):\n",
    "    if original is None:\n",
    "        return None,None\n",
    "    image = original > 0.9\n",
    "    # show_images([image])\n",
    "    numberOfIteration = (image.shape[0]+image.shape[1])//12\n",
    "    # print(image.shape, numberOfIteration)\n",
    "    # kernel = np.ones((5, 5), np.uint8)\n",
    "\n",
    "    for i in range(numberOfIteration):\n",
    "        image = binary_erosion(image)\n",
    "    for i in range(numberOfIteration):\n",
    "        image = binary_dilation(image)\n",
    "    contours = find_contours(image)\n",
    "    center = ()\n",
    "    radius = 0\n",
    "\n",
    "    contours = sorted(contours, key=cmp_to_key(compareFingers))\n",
    "    if not len(contours) > 0:\n",
    "        return None,None\n",
    "    Ymin = int(np.min(contours[-1][:, 1]))\n",
    "    Ymax = int(np.max(contours[-1][:, 1]))\n",
    "    Xmin = int(np.min(contours[-1][:, 0]))\n",
    "    Xmax = int(np.max(contours[-1][:, 0]))\n",
    "    center = ((Xmin+Xmax)//2, (Ymin+Ymax)//2)\n",
    "    radius = center[0] - Xmin\n",
    "\n",
    "    for i in range(image.shape[0]):\n",
    "        for j in range(image.shape[1]):\n",
    "            if math.dist((i, j), center) < radius:\n",
    "                original[i, j] = 0\n",
    "\n",
    "    for i in range(5):\n",
    "        original = binary_erosion(original, [[1, 1, 1] for _ in range(10)])\n",
    "    for i in range(5):\n",
    "        original = binary_dilation(original, [[1, 1, 1] for _ in range(10)])\n",
    "    show_images([original, image])\n",
    "\n",
    "    fingersCotours = find_contours(original)\n",
    "    fingersCotours = sorted(fingersCotours, key=cmp_to_key(compareFingers))\n",
    "    if len(fingersCotours) == 0:\n",
    "        return None,None\n",
    "    contour = fingersCotours[-1]\n",
    "    Ymin = int(np.min(contour[:, 1]))\n",
    "    Ymax = int(np.max(contour[:, 1]))\n",
    "    Xmin = int(np.min(contour[:, 0]))\n",
    "    Xmax = int(np.max(contour[:, 0]))\n",
    "    img = np.copy(original)\n",
    "\n",
    "    img[Xmin:Xmax, Ymin:Ymax] = 0\n",
    "    img = original.astype(int) - img.astype(int)\n",
    "    return (Xmax - Xmin), (Xmax, (Ymin+Ymax)//2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def get_info_from_fingers(fingers_info):\n",
    "    length = len(fingers_info)\n",
    "    if length == 0:\n",
    "        return None\n",
    "    if length == 1:\n",
    "        return fingers_info[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#moving mouse coordinates\n",
    "def move_mouse_by_defference(x_new, y_new):\n",
    "    x0, y0 = pg.position()\n",
    "    pg.moveTo(x0 + x_new, y0 + y_new)\n",
    "# move_mouse_by_defference(-30,150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mCanceled future for execute_request message before replies were done"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    #\n",
    "    cap = cv.VideoCapture(0)\n",
    "    cap.set(cv.CAP_PROP_BUFFERSIZE, 2)\n",
    "    currentPosition = (0, 0)\n",
    "    while True:\n",
    "        img = captureImage(cap)\n",
    "        sleep(0.1)\n",
    "        binary_image = get_image_with_skin_color(img)\n",
    "        # show_images([binary_image])\n",
    "        binary_image = enhance_image(binary_image)\n",
    "        # show_images([binary_image])\n",
    "        hand_image, hand_margin = detectHand(binary_image, img)\n",
    "        show_images([binary_image, hand_image]\n",
    "                    ) if hand_image is not None else None\n",
    "        # print(hand_image)\n",
    "        # if(hand_image is not None):\n",
    "            # print(hand_image.shape,hand_margin)\n",
    "        length, finger_center = detectFingers(hand_image)\n",
    "        if(finger_center):\n",
    "            finger_center_in_image = (finger_center[0]+hand_margin[0],finger_center[1]+hand_margin[2])\n",
    "            # print(finger_center,finger_center_in_image)\n",
    "            if(currentPosition==(0,0)):\n",
    "                currentPosition = finger_center_in_image\n",
    "            move_mouse_by_defference(\n",
    "                currentPosition[0]-finger_center_in_image[0], currentPosition[1]-finger_center_in_image[1])\n",
    "            currentPosition = finger_center_in_image\n",
    "            # print(length,finger_center)\n",
    "\n",
    "        # information = get_info_from_fingers(fingers)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}