JUST_DANCE_FINAL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7582598b-5223-4701-97dd-0e27070da7e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "import platform\n",
    "import copy\n",
    "\n",
    "%pip install -q \"openvino-dev>=2024.0.0\"\n",
    "%pip install -q opencv-python requests scipy\n",
    "\n",
    "if platform.system() != \"Windows\":\n",
    "    %pip install -q \"matplotlib>=3.4\"\n",
    "else:\n",
    "    %pip install -q \"matplotlib>=3.4,<3.7\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b555a6df-e64a-4059-b380-ed86b1173251",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "from pathlib import Path\n",
    "import sys\n",
    "import time\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import cv2\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "import openvino as ov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d661682-a679-45c7-8065-cc0a3127710e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.lib.stride_tricks import as_strided\n",
    "from decoder import OpenPoseDecoder\n",
    "\n",
    "from IPython.display import display, Javascript"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b1e5f94a-d362-4969-af8a-684c706d8df0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'openvino_notebooks' already exists and is not an empty directory.\n"
     ]
    }
   ],
   "source": [
    "# Import local modules\n",
    "\n",
    "utils_file_path = Path('../utils/notebook_utils.py')\n",
    "notebook_directory_path = Path('.')\n",
    "\n",
    "if not utils_file_path.exists():\n",
    "    !git clone --depth 1 https://github.com/igor-davidyuk/openvino_notebooks.git -b moving_data_to_cloud openvino_notebooks\n",
    "    utils_file_path = Path('./openvino_notebooks/notebooks/utils/notebook_utils.py')\n",
    "    notebook_directory_path = Path('./openvino_notebooks/notebooks/407-person-tracking-webcam/')\n",
    "\n",
    "sys.path.append(str(utils_file_path.parent))\n",
    "sys.path.append(str(notebook_directory_path))\n",
    "\n",
    "import notebook_utils as utils\n",
    "from deepsort_utils.tracker import Tracker\n",
    "from deepsort_utils.nn_matching import NearestNeighborDistanceMetric\n",
    "from deepsort_utils.detection import Detection, compute_color_for_labels, xywh_to_xyxy, xywh_to_tlwh, tlwh_to_xyxy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "96d9eeee-81c3-4ac2-81bc-592e53d78ee1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "################|| Downloading person-detection-0202 ||################\n",
      "\n",
      "========== Retrieving model/intel/person-detection-0202/FP16-INT8/person-detection-0202.xml from the cache\n",
      "\n",
      "========== Retrieving model/intel/person-detection-0202/FP16-INT8/person-detection-0202.bin from the cache\n",
      "\n",
      "################|| Downloading person-reidentification-retail-0287 ||################\n",
      "\n",
      "========== Retrieving model/intel/person-reidentification-retail-0287/person-reidentification-retail-0267.onnx from the cache\n",
      "\n",
      "========== Retrieving model/intel/person-reidentification-retail-0287/FP16-INT8/person-reidentification-retail-0287.xml from the cache\n",
      "\n",
      "========== Retrieving model/intel/person-reidentification-retail-0287/FP16-INT8/person-reidentification-retail-0287.bin from the cache\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# A directory where the model will be downloaded.\n",
    "base_model_dir = \"model\"\n",
    "precision = \"FP16-INT8\"\n",
    "# The name of the model from Open Model Zoo\n",
    "detection_model_name = \"person-detection-0202\"\n",
    "\n",
    "download_command = f\"omz_downloader \" \\\n",
    "                   f\"--name {detection_model_name} \" \\\n",
    "                   f\"--precisions {precision} \" \\\n",
    "                   f\"--output_dir {base_model_dir} \" \\\n",
    "                   f\"--cache_dir {base_model_dir}\"\n",
    "! $download_command\n",
    "\n",
    "detection_model_path = f\"model/intel/{detection_model_name}/{precision}/{detection_model_name}.xml\"\n",
    "\n",
    "\n",
    "reidentification_model_name = \"person-reidentification-retail-0287\"\n",
    "\n",
    "download_command = f\"omz_downloader \" \\\n",
    "                   f\"--name {reidentification_model_name} \" \\\n",
    "                   f\"--precisions {precision} \" \\\n",
    "                   f\"--output_dir {base_model_dir} \" \\\n",
    "                   f\"--cache_dir {base_model_dir}\"\n",
    "! $download_command\n",
    "\n",
    "reidentification_model_path = f\"model/intel/{reidentification_model_name}/{precision}/{reidentification_model_name}.xml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "00c7e5d8-9724-4395-8371-50156c1615f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model/intel/human-pose-estimation-0001/FP16-INT8/human-pose-estimation-0001.xml\n"
     ]
    }
   ],
   "source": [
    "# A directory where the model will be downloaded.\n",
    "\n",
    "\n",
    "# The name of the model from Open Model Zoo.\n",
    "model_name = \"human-pose-estimation-0001\"\n",
    "# Selected precision (FP32, FP16, FP16-INT8).\n",
    "\n",
    "\n",
    "model_path = Path(f\"{base_model_dir}/intel/{model_name}/{precision}/{model_name}.xml\")\n",
    "print(model_path)\n",
    "if not model_path.exists():\n",
    "    model_url_dir = f\"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/3/{model_name}/{precision}/\"\n",
    "    utils.download_file(model_url_dir + model_name + '.xml', model_path.name, model_path.parent)\n",
    "    utils.download_file(model_url_dir + model_name + '.bin', model_path.with_suffix('.bin').name, model_path.parent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e150ffa3-fb27-4450-979b-053ec9714363",
   "metadata": {},
   "outputs": [],
   "source": [
    "core = ov.Core()\n",
    "\n",
    "\n",
    "class Model:\n",
    "    \"\"\"\n",
    "    This class represents a OpenVINO model object.\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, model_path, batchsize=1, device=\"AUTO\"):\n",
    "        \"\"\"\n",
    "        Initialize the model object\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        model_path: path of inference model\n",
    "        batchsize: batch size of input data\n",
    "        device: device used to run inference\n",
    "        \"\"\"\n",
    "        self.model = core.read_model(model=model_path)\n",
    "        self.input_layer = self.model.input(0)\n",
    "        self.input_shape = self.input_layer.shape\n",
    "        self.height = self.input_shape[2]\n",
    "        self.width = self.input_shape[3]\n",
    "\n",
    "        for layer in self.model.inputs:\n",
    "            input_shape = layer.partial_shape\n",
    "            input_shape[0] = batchsize\n",
    "            self.model.reshape({layer: input_shape})\n",
    "        self.compiled_model = core.compile_model(model=self.model, device_name=device)\n",
    "        self.output_layer = self.compiled_model.output(0)\n",
    "\n",
    "    def predict(self, input):\n",
    "        \"\"\"\n",
    "        Run inference\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        input: array of input data\n",
    "        \"\"\"\n",
    "        result = self.compiled_model(input)[self.output_layer]\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9672996d-493e-404f-b675-264307794c2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "293d682df1824277ab3902db1bc83eed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Dropdown(description='Device:', index=2, options=('CPU', 'GPU', 'AUTO'), value='AUTO')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ipywidgets as widgets\n",
    "\n",
    "device = widgets.Dropdown(\n",
    "    options=core.available_devices + [\"AUTO\"],\n",
    "    value='AUTO',\n",
    "    description='Device:',\n",
    "    disabled=False,\n",
    ")\n",
    "\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8fbb0357-4d2d-4c45-b536-852aaa018e58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,3,256,456]\n"
     ]
    }
   ],
   "source": [
    "# Initialize OpenVINO Runtime\n",
    "\n",
    "# Read the network from a file.\n",
    "model = core.read_model(model_path)\n",
    "# Let the AUTO device decide where to load the model (you can use CPU, GPU as well).\n",
    "compiled_model_p = core.compile_model(model=model, device_name=device.value, config={\"PERFORMANCE_HINT\": \"LATENCY\"})\n",
    "\n",
    "# Get the input and output names of nodes.\n",
    "input_layer = compiled_model_p.input(0)\n",
    "output_layers = compiled_model_p.outputs\n",
    "\n",
    "# NCHW\n",
    "\n",
    "# Get the input size.\n",
    "height_p, width_p = list(input_layer.shape)[2:]\n",
    "print(input_layer.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "378e9685-a5f9-4e17-a0cb-13579b77dcd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "detector = Model(detection_model_path, device=device.value)\n",
    "# since the number of detection object is uncertain, the input batch size of reid model should be dynamic\n",
    "extractor = Model(reidentification_model_path, -1, device.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e5e48c9-39b1-4e26-b7da-2e614ef461f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5daddf57-55ba-4cf3-8502-60c0b23fa387",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('data', ['Mconv7_stage2_L1', 'Mconv7_stage2_L2'])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_layer.any_name, [o.any_name for o in output_layers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1b015c36-30f1-449e-a54d-3b2a9ecc8b03",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = OpenPoseDecoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b25d835f-749a-4505-ac1c-6be421434c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2D pooling in numpy (from: https://stackoverflow.com/a/54966908/1624463)\n",
    "def pool2d(A, kernel_size, stride, padding, pool_mode=\"max\"):\n",
    "    \"\"\"\n",
    "    2D Pooling\n",
    "\n",
    "    Parameters:\n",
    "        A: input 2D array\n",
    "        kernel_size: int, the size of the window\n",
    "        stride: int, the stride of the window\n",
    "        padding: int, implicit zero paddings on both sides of the input\n",
    "        pool_mode: string, 'max' or 'avg'\n",
    "    \"\"\"\n",
    "    # Padding\n",
    "    A = np.pad(A, padding, mode=\"constant\")\n",
    "\n",
    "    # Window view of A\n",
    "    output_shape = (\n",
    "        (A.shape[0] - kernel_size) // stride + 1,\n",
    "        (A.shape[1] - kernel_size) // stride + 1,\n",
    "    )\n",
    "    kernel_size = (kernel_size, kernel_size)\n",
    "    A_w = as_strided(\n",
    "        A,\n",
    "        shape=output_shape + kernel_size,\n",
    "        strides=(stride * A.strides[0], stride * A.strides[1]) + A.strides\n",
    "    )\n",
    "    A_w = A_w.reshape(-1, *kernel_size)\n",
    "\n",
    "    # Return the result of pooling.\n",
    "    if pool_mode == \"max\":\n",
    "        return A_w.max(axis=(1, 2)).reshape(output_shape)\n",
    "    elif pool_mode == \"avg\":\n",
    "        return A_w.mean(axis=(1, 2)).reshape(output_shape)\n",
    "\n",
    "\n",
    "# non maximum suppression\n",
    "def heatmap_nms(heatmaps, pooled_heatmaps):\n",
    "    return heatmaps * (heatmaps == pooled_heatmaps)\n",
    "\n",
    "\n",
    "# Get poses from results.\n",
    "def process_results_p(img, pafs, heatmaps):\n",
    "    \n",
    "    # This processing comes from\n",
    "    # https://github.com/openvinotoolkit/open_model_zoo/blob/master/demos/common/python/models/open_pose.py\n",
    "    pooled_heatmaps = np.array(\n",
    "        [[pool2d(h, kernel_size=3, stride=1, padding=1, pool_mode=\"max\") for h in heatmaps[0]]]\n",
    "    )\n",
    "    nms_heatmaps = heatmap_nms(heatmaps, pooled_heatmaps)\n",
    "\n",
    "    # Decode poses.\n",
    "    poses, scores = decoder(heatmaps, nms_heatmaps, pafs)\n",
    "    output_shape = list(compiled_model_p.output(index=0).partial_shape)\n",
    "    output_scale = img.shape[1] / output_shape[3].get_length(), img.shape[0] / output_shape[2].get_length()\n",
    "    # Multiply coordinates by a scaling factor.\n",
    "    poses[:, :, :2] *= output_scale\n",
    "    return poses, scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dc51d592-f2d9-40bf-a784-3f06d7a6e0d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(frame, height, width):\n",
    "    \"\"\"\n",
    "    Preprocess a single image\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    frame: input frame\n",
    "    height: height of model input data\n",
    "    width: width of model input data\n",
    "    \"\"\"\n",
    "    resized_image = cv2.resize(frame, (width, height))\n",
    "    resized_image = resized_image.transpose((2, 0, 1))\n",
    "    input_image = np.expand_dims(resized_image, axis=0).astype(np.float32)\n",
    "    return input_image\n",
    "\n",
    "\n",
    "def batch_preprocess(img_crops, height, width):\n",
    "    \"\"\"\n",
    "    Preprocess batched images\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    img_crops: batched input images\n",
    "    height: height of model input data\n",
    "    width: width of model input data\n",
    "    \"\"\"\n",
    "    img_batch = np.concatenate([\n",
    "        preprocess(img, height, width)\n",
    "        for img in img_crops\n",
    "    ], axis=0)\n",
    "    return img_batch\n",
    "\n",
    "\n",
    "def process_results(h, w, results, thresh=0.5):\n",
    "    \"\"\"\n",
    "    postprocess detection results\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    h, w: original height and width of input image\n",
    "    results: raw detection network output\n",
    "    thresh: threshold for low confidence filtering\n",
    "    \"\"\"\n",
    "    # The 'results' variable is a [1, 1, N, 7] tensor.\n",
    "    detections = results.reshape(-1, 7)\n",
    "    boxes = []\n",
    "    labels = []\n",
    "    scores = []\n",
    "    for i, detection in enumerate(detections):\n",
    "        _, label, score, xmin, ymin, xmax, ymax = detection\n",
    "        # Filter detected objects.\n",
    "        if score > thresh:\n",
    "            # Create a box with pixels coordinates from the box with normalized coordinates [0,1].\n",
    "            boxes.append(\n",
    "                [(xmin + xmax) / 2 * w, (ymin + ymax) / 2 * h, (xmax - xmin) * w, (ymax - ymin) * h]\n",
    "            )\n",
    "            labels.append(int(label))\n",
    "            scores.append(float(score))\n",
    "\n",
    "    if len(boxes) == 0:\n",
    "        boxes = np.array([]).reshape(0, 4)\n",
    "        scores = np.array([])\n",
    "        labels = np.array([])\n",
    "    return np.array(boxes), np.array(scores), np.array(labels)\n",
    "\n",
    "\n",
    "def draw_boxes(img, bbox, identities=None):\n",
    "    \"\"\"\n",
    "    Draw bounding box in original image\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    img: original image\n",
    "    bbox: coordinate of bounding box\n",
    "    identities: identities IDs\n",
    "    \"\"\"\n",
    "    for i, box in enumerate(bbox):\n",
    "        x1, y1, x2, y2 = [int(i) for i in box]\n",
    "        # box text and bar\n",
    "        id = int(identities[i]) if identities is not None else 0\n",
    "        color = compute_color_for_labels(id)\n",
    "        label = '{}{:d}'.format(\"\", id)\n",
    "        t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_PLAIN, 2, 2)[0]\n",
    "        cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)\n",
    "        cv2.rectangle(\n",
    "            img, (x1, y1), (x1 + t_size[0] + 3, y1 + t_size[1] + 4), color, -1)\n",
    "        cv2.putText(\n",
    "            img,\n",
    "            label,\n",
    "            (x1, y1 + t_size[1] + 4),\n",
    "            cv2.FONT_HERSHEY_PLAIN,\n",
    "            1.6,\n",
    "            [255, 255, 255],\n",
    "            2\n",
    "        )\n",
    "    return img\n",
    "\n",
    "\n",
    "def cosin_metric(x1, x2):\n",
    "    \"\"\"\n",
    "    Calculate the consin distance of two vector\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    x1, x2: input vectors\n",
    "    \"\"\"\n",
    "    return np.dot(x1, x2) / (np.linalg.norm(x1) * np.linalg.norm(x2))\n",
    "\n",
    "\n",
    "\n",
    "def calculate_angle(x1, y1, x2, y2, x3, y3):\n",
    "    # 세 점을 포함하는 두 벡터 계산\n",
    "    vector1 = np.array([x2 - x1, y2 - y1])\n",
    "    vector2 = np.array([x3 - x2, y3 - y2])\n",
    "    \n",
    "    # 각 선분의 방향 벡터를 정규화\n",
    "    norm_vector1 = -vector1 / np.linalg.norm(vector1)\n",
    "    norm_vector2 = vector2 / np.linalg.norm(vector2)\n",
    "        \n",
    "    # 두 벡터 사이의 각도 계산 (라디안 단위)\n",
    "    dot_product = np.dot(norm_vector1, norm_vector2)\n",
    "    angle_rad = np.arccos(np.clip(dot_product, -1.0, 1.0)) # arc-cosine의 값이 -1과 1 사이에 있어야 합니다.\n",
    "    \n",
    "    # 라디안을 도 단위로 변환\n",
    "    angle_deg = np.degrees(angle_rad)\n",
    "    \n",
    "    if np.isnan(angle_deg):\n",
    "        angle_deg = 0\n",
    "        \n",
    "    angle_deg = int(angle_deg)\n",
    "    return angle_deg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c7159700-2fbd-41b6-8092-346c3b7c5793",
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = ((255, 0, 0), (255, 0, 255), (170, 0, 255), (255, 0, 85), (255, 0, 170), (85, 255, 0),\n",
    "          (255, 170, 0), (0, 255, 0), (255, 255, 0), (0, 255, 85), (170, 255, 0), (0, 85, 255),\n",
    "          (0, 255, 170), (0, 0, 255), (0, 255, 255), (85, 0, 255), (0, 170, 255))\n",
    "\n",
    "default_skeleton = ((15, 13), (13, 11), (16, 14), (14, 12), (11, 12), (5, 11), (6, 12), (5, 6), (5, 7),\n",
    "                    (6, 8), (7, 9), (8, 10), (1, 2), (0, 1), (0, 2), (1, 3), (2, 4), (3, 5), (4, 6))\n",
    "\n",
    "\n",
    "def draw_poses(img, poses, point_score_threshold, skeleton=default_skeleton):\n",
    "    font = cv2.FONT_HERSHEY_SIMPLEX  # 텍스트 폰트\n",
    "    font_scale = 0.5  # 텍스트 크기 배율\n",
    "    font_color = (255, 255, 255)  # 텍스트 색상 (BGR 형식)\n",
    "    angle_list = [0, 0, 0, 0, 0, 0, 0, 0]\n",
    "    \n",
    "    if poses.size == 0:\n",
    "        return img, angle_list\n",
    "    \n",
    "    img_limbs = np.copy(img)\n",
    "    for pose in poses:\n",
    "        points = pose[:, :2].astype(np.int32)\n",
    "\n",
    "        angle_l_shoul = calculate_angle(points[3][0], points[3][1], points[5][0], points[5][1], points[7][0], points[7][1])\n",
    "        #cv2.putText(img, f\"{angle_l_shoul}\", (points[5][0]+10, points[5][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_r_shoul = calculate_angle(points[4][0], points[4][1], points[6][0], points[6][1], points[8][0], points[8][1])\n",
    "        #cv2.putText(img, f\"{angle_r_shoul}\", (points[6][0]+10, points[6][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_l_elbow = calculate_angle(points[5][0], points[5][1], points[7][0], points[7][1], points[9][0], points[9][1])\n",
    "        #cv2.putText(img, f\"{angle_l_elbow}\", (points[7][0]+10, points[7][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_r_elbow = calculate_angle(points[6][0], points[6][1], points[8][0], points[8][1], points[10][0], points[10][1])\n",
    "        #cv2.putText(img, f\"{angle_r_elbow}\", (points[8][0]+10, points[8][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_l_hip = calculate_angle(points[5][0], points[5][1], points[11][0], points[11][1], points[13][0], points[13][1])\n",
    "        #cv2.putText(img, f\"{angle_l_hip}\", (points[11][0]+10, points[11][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_r_hip = calculate_angle(points[6][0], points[6][1], points[12][0], points[12][1], points[14][0], points[14][1])\n",
    "        #cv2.putText(img, f\"{angle_r_hip}\", (points[12][0]+10, points[12][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "    \n",
    "        angle_l_knee = calculate_angle(points[11][0], points[11][1], points[13][0], points[13][1], points[15][0], points[15][1])\n",
    "        #cv2.putText(img, f\"{angle_l_knee}\", (points[13][0]+10, points[13][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "        \n",
    "        angle_r_knee = calculate_angle(points[12][0], points[12][1], points[14][0], points[14][1], points[16][0], points[16][1])\n",
    "        #cv2.putText(img, f\"{angle_r_knee}\", (points[14][0]+10, points[14][1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (100,255,0), 1)\n",
    "\n",
    "        angle_list = [angle_l_shoul, angle_r_shoul, angle_l_elbow, angle_r_elbow, angle_l_hip, angle_r_hip, angle_l_knee, angle_r_knee]\n",
    "        \n",
    "        points_scores = pose[:, 2]\n",
    "        # Draw joints.\n",
    "        for i, (p, v) in enumerate(zip(points, points_scores)):\n",
    "            if v > point_score_threshold:\n",
    "                cv2.circle(img, tuple(p), 1, colors[i], 2)\n",
    "                # cv2.putText(img, f\"({p[0]}, {p[1]})\", (p[0] + 10, p[1] - 10), font, font_scale, font_color, 1)\n",
    "            \n",
    "        # Draw limbs.\n",
    "        for i, j in skeleton:\n",
    "            if points_scores[i] > point_score_threshold and points_scores[j] > point_score_threshold:\n",
    "                cv2.line(img_limbs, tuple(points[i]), tuple(points[j]), color=colors[j], thickness=4)\n",
    "                \n",
    "    cv2.addWeighted(img, 0.9, img_limbs, 0.1, 0, dst=img)\n",
    "    \n",
    "    return img, angle_list\n",
    "\n",
    "\n",
    "def show_popup_message(message):\n",
    "    javascript_code = f\"\"\"\n",
    "        alert(\"{message}\");\n",
    "    \"\"\"\n",
    "    display(Javascript(javascript_code))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d2f681c0-abeb-4e11-8d92-fb6ae18910f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    \n",
    "\n",
    "        # while True:\n",
    "        #     h, w = frame.shape[:2]\n",
    "        #     input_image = preprocess(frame, detector.height, detector.width)\n",
    "\n",
    "        #     # Measure processing time.\n",
    "        #     start_time = time.time()\n",
    "        #     # Get the results.\n",
    "        #     output = detector.predict(input_image)\n",
    "        #     stop_time = time.time()\n",
    "        #     processing_times.append(stop_time - start_time)\n",
    "        #     if len(processing_times) > 200:\n",
    "        #         processing_times.popleft()\n",
    "\n",
    "        #     _, f_width = frame.shape[:2]\n",
    "            \n",
    "        #     # Mean processing time [ms].\n",
    "        #     processing_time = np.mean(processing_times) * 1100\n",
    "        #     fps = 1000 / processing_time\n",
    "\n",
    "        #     # Get poses from detection results.\n",
    "        #     bbox_xywh, score, label = process_results(h, w, results=output)\n",
    "            \n",
    "        #     img_crops = []\n",
    "        #     for box in bbox_xywh:\n",
    "        #         x1, y1, x2, y2 = xywh_to_xyxy(box, h, w)\n",
    "        #         img = frame[y1:y2, x1:x2]\n",
    "        #         img_crops.append(img)\n",
    "\n",
    "        #     # Get reidentification feature of each person.\n",
    "        #     if img_crops:\n",
    "        #         # preprocess\n",
    "        #         img_batch = batch_preprocess(img_crops, extractor.height, extractor.width)\n",
    "        #         features = extractor.predict(img_batch)\n",
    "        #     else:\n",
    "        #         features = np.array([])\n",
    "\n",
    "        #     # Wrap the detection and reidentification results together\n",
    "            \n",
    "        #     bbox_tlwh = xywh_to_tlwh(bbox_xywh)\n",
    "        #     detections = [\n",
    "        #         Detection(bbox_tlwh[i], features[i])\n",
    "        #         for i in range(features.shape[0])\n",
    "        #     ]\n",
    "\n",
    "        #     # predict the position of tracking target \n",
    "        #     tracker.predict()\n",
    "\n",
    "        #     # update tracker\n",
    "        #     tracker.update(detections)\n",
    "\n",
    "        #     # update bbox identities\n",
    "        #     outputs = []\n",
    "        #     for track in tracker.tracks:\n",
    "        #         if not track.is_confirmed() or track.time_since_update > 1:\n",
    "        #             continue\n",
    "        #         box = track.to_tlwh()\n",
    "        #         x1, y1, x2, y2 = tlwh_to_xyxy(box, h, w)\n",
    "        #         track_id = track.track_id\n",
    "        #         outputs.append(np.array([x1, y1, x2, y2, track_id], dtype=np.int32))\n",
    "        #     if len(outputs) > 0:\n",
    "        #         outputs = np.stack(outputs, axis=0)\n",
    "\n",
    "        #     # draw box for visualization\n",
    "        #     if len(outputs) > 0:\n",
    "        #         bbox_tlwh = []\n",
    "        #         bbox_xyxy = outputs[:, :4]\n",
    "        #         identities = outputs[:, -1]\n",
    "        #         frame = draw_boxes(frame, bbox_xyxy, identities)\n",
    "\n",
    "        #     cv2.putText(\n",
    "        #         img=frame,\n",
    "        #         text=f\"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)\",\n",
    "        #         org=(20, 40),\n",
    "        #         fontFace=cv2.FONT_HERSHEY_COMPLEX,\n",
    "        #         fontScale=f_width / 1000,\n",
    "        #         color=(0, 0, 255),\n",
    "        #         thickness=1,\n",
    "        #         lineType=cv2.LINE_AA,\n",
    "        #     )\n",
    "            \n",
    "        #     if use_popup:\n",
    "        #         resized_frame1 = cv2.resize(frame1, (700, 450), interpolation=cv2.INTER_LINEAR)\n",
    "        #         stacked_frame = np.vstack((frame,resized_frame1))\n",
    "        #         cv2.imshow(winname=title, mat=stacked_frame)\n",
    "        #         key = cv2.waitKey(1)\n",
    "        #         # escape = 27\n",
    "        #         if key == 27:\n",
    "        #             # 파일 이름 변경 함수 호출\n",
    "                \n",
    "        #             break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5d51a26d-9451-483e-88ea-b99637689e44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main processing function to run pose estimation.\n",
    "def run_pose_estimation(source=0, flip=False, use_popup=False, skip_first_frames=0):\n",
    "    pafs_output_key = compiled_model_p.output(\"Mconv7_stage2_L1\")\n",
    "    heatmaps_output_key = compiled_model_p.output(\"Mconv7_stage2_L2\")\n",
    "    player = None\n",
    "    player1 = None\n",
    "    score_list=[]\n",
    "    try:\n",
    "        # Create a video player to play with target fps.\n",
    "        player = utils.VideoPlayer(source,size=(700, 450), flip=flip, fps=24)\n",
    "        player1 = utils.VideoPlayer(0, size=(700, 450), flip=True, fps=24)\n",
    "        # Start capturing.\n",
    "        player.start()\n",
    "        player1.start()\n",
    "        if use_popup:\n",
    "            title = \"Press ESC to Exit windows0\"\n",
    "            title1 = \"Press ESC to Exit windows0 1\"\n",
    "            title2 = \"Press ESC to Exit windows0 2\"\n",
    "            \n",
    "           \n",
    "            cv2.namedWindow(title, cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE)\n",
    "            cv2.namedWindow(title1, cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE)\n",
    "            \n",
    "            \n",
    "\n",
    "\n",
    "        processing_times = collections.deque()\n",
    "\n",
    "        while True:\n",
    "            #############################person tracking##############################################\n",
    "            #\n",
    "            frame = player.next()\n",
    "            frame1 = player1.next()\n",
    "            \n",
    "            if frame is None:\n",
    "                print(\"Source ended\")\n",
    "                break\n",
    "                \n",
    "            if frame1 is None:\n",
    "                print(\"Source ended\")\n",
    "                break\n",
    "\n",
    "            scale = 1280 / max(frame.shape)\n",
    "            scale1 = 1280 / max(frame1.shape)\n",
    "            if scale < 1:\n",
    "                frame = cv2.resize(frame, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA)\n",
    "                \n",
    "            if scale1 < 1:\n",
    "                frame1 = cv2.resize(frame1, None, fx=scale, fy=scale, interpolation=cv2.INTER_AREA)                \n",
    "            \n",
    "            \n",
    "                \n",
    "            h, w = frame.shape[:2]\n",
    "            \n",
    "            \n",
    "\n",
    "            \n",
    "            input_image = preprocess(frame, detector.height, detector.width)\n",
    "            input_image1 = preprocess(frame1, detector.height, detector.width)\n",
    "            \n",
    "           \n",
    "            # Measure processing time.\n",
    "            start_time = time.time()\n",
    "            \n",
    "            \n",
    "            # Get the results.\n",
    "            output = detector.predict(input_image)\n",
    "            output1 = detector.predict(input_image1)\n",
    "            \n",
    "            stop_time = time.time()\n",
    "            processing_times.append(stop_time - start_time)\n",
    "            if len(processing_times) > 200:\n",
    "                processing_times.popleft()\n",
    "\n",
    "            _, f_width = frame.shape[:2]\n",
    "            # _, f_width1 = frame1.shape[:2]\n",
    "            \n",
    "            # Mean processing time [ms].\n",
    "            processing_time = np.mean(processing_times) * 1100\n",
    "            fps = 1000 / processing_time\n",
    "\n",
    "            # Get poses from detection results.\n",
    "            bbox_xywh, score, _ = process_results(h, w, results=output)\n",
    "            bbox_xywh1, score1, _ = process_results(h, w, results=output1)\n",
    "            \n",
    "            \n",
    "\n",
    "            \n",
    "            if len(bbox_xywh):\n",
    "                x1, y1, x2, y2 = xywh_to_xyxy(bbox_xywh[0], h, w)\n",
    "                width = (x2+x1)\n",
    "                height = (y2+y1)\n",
    "                CX = int(width/2)\n",
    "                CY = int(height/2)\n",
    "                \n",
    "                y1 = CY - int(0.8*height_p)\n",
    "                y2 = CY + int(0.6*height_p)\n",
    "                \n",
    "                x1 = CX - int(0.3*width_p)\n",
    "                x2 = CX + int(0.3*width_p)\n",
    "                \n",
    "                #y1 = y1 + CY - int(height_p/2)\n",
    "                #y2 = y2 - CY + int(height_p/2)\n",
    "                #x1 = x1 + CX - int(width_p/2)\n",
    "                #x2 = x2 - CX + int(width_p/2)\n",
    "                traking_image = copy.deepcopy(frame[y1:y2, x1:x2])\n",
    "                #print(traking_image.shape,end='\\r')\n",
    "            else : \n",
    "                continue \n",
    "\n",
    "            \n",
    "            # if len(bbox_xywh1):\n",
    "            #     x11, y11, x21, y21 = xywh_to_xyxy(bbox_xywh1[0], h, w)\n",
    "            #     width1 = (x21+x11)\n",
    "            #     height1 = (y21+y11)\n",
    "            #     CX = int(width1/2)\n",
    "            #     CY = int(height1/2)\n",
    "                \n",
    "            #     y11 = CY - int(0.8*height_p)\n",
    "            #     y21 = CY + int(0.6*height_p)\n",
    "                \n",
    "            #     x11 = CX - int(0.3*width_p)\n",
    "            #     x21 = CX + int(0.3*width_p)\n",
    "                \n",
    "            #     #y1 = y1 + CY - int(height_p/2)\n",
    "            #     #y2 = y2 - CY + int(height_p/2)\n",
    "            #     #x1 = x1 + CX - int(width_p/2)\n",
    "            #     #x2 = x2 - CX + int(width_p/2)\n",
    "            #     traking_image1 = frame1[y11:y21, x11:x21]\n",
    "            #     #print(traking_image.shape,end='\\r')\n",
    "            # else : \n",
    "            traking_image1 = frame1\n",
    "                \n",
    "           \n",
    "\n",
    "            input_img = cv2.resize(traking_image, (width_p, height_p), interpolation=cv2.INTER_AREA)\n",
    "            input_img1 = cv2.resize(traking_image1, (width_p, height_p), interpolation=cv2.INTER_AREA)\n",
    "            \n",
    "            input_img = input_img.transpose((2,0,1))[np.newaxis, ...]\n",
    "            input_img1 = input_img1.transpose((2,0,1))[np.newaxis, ...]\n",
    "\n",
    "            # Measure processing time.\n",
    "            \n",
    "            # Get results.\n",
    "            results_p = compiled_model_p([input_img])\n",
    "            results1_p = compiled_model_p([input_img1])\n",
    "            \n",
    "\n",
    "            pafs = results_p[pafs_output_key]\n",
    "            heatmaps = results_p[heatmaps_output_key]\n",
    "\n",
    "            pafs1 = results1_p[pafs_output_key]\n",
    "            heatmaps1 = results1_p[heatmaps_output_key]\n",
    "\n",
    "\n",
    "            \n",
    "            # Get poses from network results.\n",
    "            poses, scores = process_results_p(traking_image, pafs, heatmaps)\n",
    "            poses1, scores1 = process_results_p(traking_image1, pafs1, heatmaps1)\n",
    "\n",
    "            \n",
    "            # Draw poses on a frame.\n",
    "            \n",
    "            traking_image, answer_list = draw_poses(traking_image, poses, 0.1)\n",
    "            traking_image1, player_list = draw_poses(traking_image1, poses1, 0.1)\n",
    "            # frame1 = draw_poses(traking_image1, poses1, 0.1)\n",
    "            percent = 0\n",
    "            total_percent = 0\n",
    "            for i in range(8):\n",
    "                percent = 100 - abs((answer_list[i] - player_list[i])/180 * 100)\n",
    "                total_percent += percent\n",
    "            percent_avg = total_percent/8\n",
    "            cv2.putText(frame1, f\"{int(percent_avg)}%\", (600, 400), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)\n",
    "\n",
    "            score_list.append(percent_avg)\n",
    "            \n",
    "            \n",
    "\n",
    "            _, f_width = traking_image.shape[:2]\n",
    "            # _, f_width1 = frame1.shape[:2]\n",
    "            # mean processing time [ms]\n",
    "            \n",
    "            \n",
    "            cv2.putText(frame, f\"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)\", (20, 40),\n",
    "                        cv2.FONT_HERSHEY_COMPLEX, f_width / 1000, (0, 0, 255), 1, cv2.LINE_AA)\n",
    "            # cv2.putText(frame1, f\"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)\", (20, 40),\n",
    "            #             cv2.FONT_HERSHEY_COMPLEX, f_width1 / 1000, (0, 0, 255), 1, cv2.LINE_AA)\n",
    "\n",
    "            # Use this workaround if there is flickering.\n",
    "            if use_popup:\n",
    "\n",
    "                # print(frame.shape)\n",
    "                # print(frame1.shape)\n",
    "                \n",
    "                stacked_array = np.vstack((frame, frame1))\n",
    "                \n",
    "                \n",
    "                cv2.imshow(title, stacked_array)\n",
    "                key = cv2.waitKey(1)\n",
    "                \n",
    "                \n",
    "\n",
    "                # traking_image = np.vstack((traking_image, traking_image1))\n",
    "                \n",
    "                cv2.imshow(title1, traking_image)\n",
    "                key = cv2.waitKey(1)\n",
    "                \n",
    "                \n",
    "                \n",
    "                # escape = 27\n",
    "                if key == 27:\n",
    "                    break\n",
    "            else:\n",
    "                # Encode numpy array to jpg.\n",
    "                _, encoded_img = cv2.imencode(\".jpg\", frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90])\n",
    "                # Create an IPython image.\n",
    "                i = display.Image(data=encoded_img)\n",
    "                # Display the image in this notebook.\n",
    "                display.clear_output(wait=True)\n",
    "                display.display(i)\n",
    "    # ctrl-c\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Interrupted\")\n",
    "    # any different error\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "    finally:\n",
    "        print(\"finally\")\n",
    "        if player is not None:\n",
    "            # Stop capturing.\n",
    "            player.stop()\n",
    "            player1.stop()\n",
    "        if len(score_list) != 0:                \n",
    "            score = sum(score_list)/(len(score_list)+0.0001)\n",
    "            show_popup_message(f\"score : {int(score)}\")\n",
    "                        # cv2.putText(frame1, f\"score : {int(score)}\", (600, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)\n",
    "            time.sleep(5)\n",
    "        if use_popup:\n",
    "            \n",
    "            cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cee9d58c-9ca7-4270-a888-5fc9132f31ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_24195/3967060209.py:118: RuntimeWarning: invalid value encountered in divide\n",
      "  norm_vector2 = vector2 / np.linalg.norm(vector2)\n",
      "/tmp/ipykernel_24195/3967060209.py:117: RuntimeWarning: invalid value encountered in divide\n",
      "  norm_vector1 = -vector1 / np.linalg.norm(vector1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finally\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        alert(\"score : 40\");\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# video_file='./data/m.mp4'\n",
    "# source = video_file\n",
    "# additional_options = {\"skip_first_frames\": 500}\n",
    "# run_person_tracking_pose_est(source=source,source1=0, flip=False, use_popup=True,**additional_options)\n",
    "USE_WEBCAM = False\n",
    "cam_id = 0\n",
    "video_file = \"./m.mp4\"\n",
    "source = cam_id if USE_WEBCAM else video_file\n",
    "\n",
    "additional_options = {\"skip_first_frames\": 500} if not USE_WEBCAM else {}\n",
    "run_pose_estimation(source=source, flip=False, use_popup=True,**additional_options)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1972a381-3ad4-4f13-84b5-6880774148f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f91b84f2-67b3-4bcc-9991-d2bc82cc9e23",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}