AI_mini_project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dlib\n",
    "import cv2\n",
    "import numpy as np\n",
    "import imutils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance as dist\n",
    "from imutils import face_utils\n",
    "from threading import Thread\n",
    "from gtts import gTTS\n",
    "from playsound import playsound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data/images/left04.jpg', 'data/images/left06.jpg', 'data/images/left09.jpg', 'data/images/1_joey-friends.jpg', 'data/images/left07.jpg', 'data/images/left05.jpg', 'data/images/left01.jpg', 'data/images/left03.jpg', 'data/images/left14.jpg', 'data/images/joye.jpg', 'data/images/left11.jpg', 'data/images/left12.jpg', 'data/images/left.jpg', 'data/images/left08.jpg', 'data/images/left13.jpg', 'data/images/left02.jpg']\n",
      "[[515.51273861   0.         341.23697124]\n",
      " [  0.         515.48750138 234.27066711]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-2.04120057e-01 -1.03446509e+00  2.31949959e-03 -2.67680902e-03\n",
      "   5.40565638e+00]]\n",
      "[[568.84318872   0.         305.45921311]\n",
      " [  0.         576.68385278 235.16080587]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.3417715   0.21139459  0.00077113  0.01687714 -0.20818868]]\n",
      "[[564.3049453    0.         313.66492301]\n",
      " [  0.         570.56795432 234.49413053]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.34029736  0.21984852  0.00085473  0.01459844 -0.22463906]]\n",
      "[[534.7153646    0.         341.38689806]\n",
      " [  0.         534.72248006 233.00747971]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.30164742  0.14575715  0.00162192  0.00075163 -0.03888059]]\n",
      "[[536.1654838    0.         339.68318527]\n",
      " [  0.         536.42194312 232.02199755]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.30514469  0.17465636  0.00148305  0.00050148 -0.08162698]]\n",
      "[[535.85548433   0.         338.57682189]\n",
      " [  0.         536.22252605 232.38402334]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.30735852  0.20332633  0.00157274  0.00033222 -0.14580925]]\n",
      "[[533.42598789   0.         339.56394825]\n",
      " [  0.         533.52154873 232.10229326]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-0.29764546  0.16989767  0.00132281 -0.00042234 -0.09912497]]\n",
      "[[533.88098765   0.         340.61882024]\n",
      " [  0.         533.89063163 232.58823887]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-2.94696667e-01  1.36350586e-01  1.27621902e-03 -2.64900046e-04\n",
      "  -2.54867511e-02]]\n",
      "[[533.90816192   0.         341.75816991]\n",
      " [  0.         533.97266482 233.41657718]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-2.97784890e-01  1.55533943e-01  1.31343706e-03 -1.59948631e-04\n",
      "  -6.53103064e-02]]\n",
      "[[534.45538965   0.         341.3985349 ]\n",
      " [  0.         534.49969739 232.63356187]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-2.93773517e-01  1.21857473e-01  1.34270603e-03  1.42171054e-06\n",
      "   2.43413951e-03]]\n",
      "[[534.07088623   0.         341.53407106]\n",
      " [  0.         534.11914798 232.94565222]\n",
      " [  0.           0.           1.        ]]\n",
      "[[-2.92971621e-01  1.07706887e-01  1.31038490e-03 -3.11023008e-05\n",
      "   4.34799136e-02]]\n"
     ]
    }
   ],
   "source": [
    "# https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_calib3d/py_calibration/py_calibration.html\n",
    "\n",
    "import glob\n",
    "\n",
    "# termination criteria\n",
    "criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)\n",
    "\n",
    "# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)\n",
    "objp = np.zeros((6 * 7, 3), np.float32)\n",
    "objp[:, :2] = np.mgrid[0: 7, 0: 6].T.reshape(-1, 2)\n",
    "\n",
    "# Arrays to store object points and image points from all the images.\n",
    "objpoints = []  # 3d point in real world space\n",
    "imgpoints = []  # 2d points in image plane.\n",
    "\n",
    "images = glob.glob('data/images/*.jpg')\n",
    "print(images)\n",
    "\n",
    "for image in images:\n",
    "    img = cv2.imread(image)\n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "    ret, corners = cv2.findChessboardCorners(gray, (7, 6), None)\n",
    "\n",
    "    if ret:\n",
    "        objpoints.append(objp)\n",
    "\n",
    "        corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)\n",
    "        imgpoints.append(corners2)\n",
    "\n",
    "        img = cv2.drawChessboardCorners(img, (7, 6), corners2, ret)\n",
    "        cv2.imshow('img', img)\n",
    "        cv2.waitKey(500)\n",
    "\n",
    "        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)\n",
    "        print(mtx)\n",
    "        print(dist)\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(\"shape_predictor_68_face_landmarks.dat\")\n",
    "detector1 = cv2.CascadeClassifier(\"haarcascade_frontalface_default.xml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ref3DModel():\n",
    "    modelPoints = [[0.0,0.0,0.0],\n",
    "                   [0.0,-330.0,-65.0],\n",
    "                   [-255.0,170.0,-135.0],\n",
    "                   [225.0,170.0,-135.0],\n",
    "                   [-150.0,-150.0,-125.0],\n",
    "                   [150.0,-150.0,-125.0]]\n",
    "    return np.array(modelPoints,dtype=np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ref2DImagePoints(shape):\n",
    "    imagePoints = [[shape.part(30).x,shape.part(30).y],\n",
    "                   [shape.part(8).x,shape.part(8).y],\n",
    "                   [shape.part(36).x,shape.part(36).y],\n",
    "                   [shape.part(45).x,shape.part(45).y],\n",
    "                   [shape.part(48).x,shape.part(48).y],\n",
    "                   [shape.part(54).x,shape.part(54).y]]\n",
    "    return np.array(imagePoints,dtype=np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def CameraMatrix(fl,center):\n",
    "#     cameraMatrix = [[fl,1,center[0]],\n",
    "#                     [0,fl,centre[1]],\n",
    "#                     [0,0,1]]\n",
    "#     return np.array(cameraMatrix,dtype=np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drawPolyline(img,shapes,start,end,isClosed=False):\n",
    "    points = []\n",
    "    for i in range(start,end+1):\n",
    "        point = [shapes.part(i).x,shapes.part(i).y]\n",
    "        points.append(point)\n",
    "    points = np.array(points,dtype=np.float32)\n",
    "    cv2.polylines(img,np.int32([points]),isClosed,(255,80,0),thickness=1,lineType=cv2.LINE_8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw(img,shapes):\n",
    "    drawPolyline(img, shapes, 0, 16)\n",
    "    drawPolyline(img, shapes, 17, 21)\n",
    "    drawPolyline(img, shapes, 22, 26)\n",
    "    drawPolyline(img, shapes, 27, 30)\n",
    "    drawPolyline(img, shapes, 30, 35, True)\n",
    "    drawPolyline(img, shapes, 36, 41, True)\n",
    "    drawPolyline(img, shapes, 42, 47, True)\n",
    "    drawPolyline(img, shapes, 48, 59, True)\n",
    "    drawPolyline(img, shapes, 60, 67, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "EYE_AR_THRESH = 0.3\n",
    "EYE_AR_CONSEC_FRAMES = 30\n",
    "YAWN_THRESH = 20\n",
    "alarm_status = False\n",
    "alarm_status2 = False\n",
    "saying = False\n",
    "COUNTER = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def alarm(m):\n",
    "    while alarm_status:\n",
    "        print(\"Call\")\n",
    "        myText = m\n",
    "        language = 'en'\n",
    "        output = gTTS(text=myText,lang=language,slow=False)\n",
    "        output.save(\"output.mp3\")\n",
    "        playsound(\"output.mp3\")\n",
    "    \n",
    "    if alarm_status2:\n",
    "        print(\"Call\")\n",
    "        saying = True\n",
    "        myText = m\n",
    "        language = 'en'\n",
    "        output = gTTS(text=myText,lang=language,slow=False)\n",
    "        output.save(\"output.mp3\")\n",
    "        playsound(\"output.mp3\")\n",
    "        saying = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eye_aspect_ratio(eye):\n",
    "    A = np.linalg.norm(eye[1]-eye[5])\n",
    "    B = np.linalg.norm(eye[2]-eye[4])\n",
    "    C = np.linalg.norm(eye[0]-eye[3])\n",
    "    \n",
    "    ear = (A+B)/(2.0*C)\n",
    "    \n",
    "    return ear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def final_ear(shape):\n",
    "    (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS[\"left_eye\"]\n",
    "    (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS[\"right_eye\"]\n",
    "    \n",
    "    leftEye = shape[lStart:lEnd]\n",
    "    rightEye = shape[rStart:rEnd]\n",
    "    \n",
    "    leftEAR = eye_aspect_ratio(leftEye)\n",
    "    rightEAR = eye_aspect_ratio(rightEye)\n",
    "    \n",
    "    ear = (leftEAR+rightEAR)/2.0\n",
    "    return (ear,leftEye,rightEye)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lip_distance(shape):\n",
    "    top_lip = shape[50:53]\n",
    "    top_lip = np.concatenate((top_lip, shape[61:64]))\n",
    "    \n",
    "    low_lip = shape[56:59]\n",
    "    low_lip = np.concatenate((low_lip, shape[65:68]))\n",
    "    \n",
    "    top_mean = np.mean(top_lip,axis=0)\n",
    "    low_mean = np.mean(low_lip,axis=0)\n",
    "    \n",
    "    distance = abs(top_mean[1]-low_mean[1])\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Call\n",
      "Call\n"
     ]
    }
   ],
   "source": [
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "while True:\n",
    "    GAZE = \"Face Not Found\"\n",
    "    ret,img = cap.read()\n",
    "    \n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    rects = detector1.detectMultiScale(gray, scaleFactor=1.1,minNeighbors=5, minSize=(30, 30),flags=cv2.CASCADE_SCALE_IMAGE)\n",
    "\n",
    "    #for rect in rects:\n",
    "    for (x, y, w, h) in rects:\n",
    "        rect = dlib.rectangle(int(x), int(y), int(x + w),int(y + h))\n",
    "        \n",
    "        shape = predictor(gray, rect)\n",
    "        shape = face_utils.shape_to_np(shape)\n",
    "\n",
    "        eye = final_ear(shape)\n",
    "        ear = eye[0]\n",
    "        leftEye = eye [1]\n",
    "        rightEye = eye[2]\n",
    "        \n",
    "#         leftEyeHull = cv2.convexHull(leftEye)\n",
    "#         rightEyeHull = cv2.convexHull(rightEye)\n",
    "#         cv2.drawContours(img, [leftEyeHull], -1, (0, 0, 255), 1)\n",
    "#         cv2.drawContours(, [rightEyeHull], -1, (0, 0, 255), 1)\n",
    "\n",
    "        if ear < EYE_AR_THRESH:\n",
    "            COUNTER += 1\n",
    "\n",
    "            if COUNTER >= EYE_AR_CONSEC_FRAMES:\n",
    "                if alarm_status == False:\n",
    "                    alarm_status = True\n",
    "                    t = Thread(target=alarm, args=('wake up sid',))\n",
    "                    t.deamon = True\n",
    "                    t.start()\n",
    "\n",
    "                cv2.putText(img, \"DROWSINESS ALERT!\", (10, 30),\n",
    "                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "        else:\n",
    "            COUNTER = 0\n",
    "            alarm_status = False\n",
    "     \n",
    "    faces = detector(cv2.cvtColor(img,cv2.COLOR_BGR2RGB),0)\n",
    "    face3Dmodel = ref3DModel()\n",
    "    \n",
    "    for face in faces:\n",
    "        shape = predictor(cv2.cvtColor(img,cv2.COLOR_BGR2RGB),face)\n",
    "        \n",
    "        draw(img,shape)\n",
    "        \n",
    "        shape1 = face_utils.shape_to_np(shape)\n",
    "\n",
    "        distance = lip_distance(shape1)\n",
    "\n",
    "        refImgPts = ref2DImagePoints(shape)\n",
    "        \n",
    "        height,width,channels = img.shape\n",
    "#         focalLength = args.focal * width\n",
    "#         cameraMatrix = cameraMatrix(focalLength,(height/2,width/2))\n",
    "        \n",
    "        mdists = np.zeros((4,1),dtype=np.float64)\n",
    "        \n",
    "        success,rotationVector,translationVector = cv2.solvePnP(face3Dmodel,refImgPts,mtx,mdists)\n",
    "        \n",
    "        noseEndPoints3D = np.array([[0,0,1000.0]],dtype=np.float64)\n",
    "        noseEndPoint2D,jacobian = cv2.projectPoints(noseEndPoints3D, rotationVector, translationVector, mtx, mdists)\n",
    "        \n",
    "        rmat, jac = cv2.Rodrigues(rotationVector)\n",
    "        angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)\n",
    "        \n",
    "        if angles[1] < -15:\n",
    "            GAZE = \"Looking: Left\"\n",
    "        elif angles[1] > 15:\n",
    "            GAZE = \"Looking: Right\"\n",
    "        else:\n",
    "            GAZE = \"Forward\"\n",
    "            \n",
    "#         if ear < EYE_AR_THRESH:\n",
    "#             COUNTER += 1\n",
    "\n",
    "#             if COUNTER >= EYE_AR_CONSEC_FRAMES:\n",
    "#                 if alarm_status == False:\n",
    "#                     alarm_status = True\n",
    "#                     t = Thread(target=alarm, args=('wake up sir',))\n",
    "#                     t.deamon = True\n",
    "#                     t.start()\n",
    "\n",
    "#                 cv2.putText(img, \"DROWSINESS ALERT!\", (10, 30),\n",
    "#                             cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "#         else:\n",
    "#             COUNTER = 0\n",
    "#             alarm_status = False\n",
    "\n",
    "        if (distance > YAWN_THRESH):\n",
    "                cv2.putText(img, \"Yawn Alert\", (10, 30),\n",
    "                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "        else:\n",
    "            alarm_status2 = False\n",
    "\n",
    "            \n",
    "    cv2.putText(img,GAZE,(20,20),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,80),2)\n",
    "    cv2.imshow(\"Head Pose\",img)\n",
    "    \n",
    "    key = cv2.waitKey(10) & 0xFF\n",
    "    if key == 27:\n",
    "        break\n",
    "        \n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}