Skip to content

Commit

Permalink
Integrated to classifiers into one
Browse files Browse the repository at this point in the history
  • Loading branch information
@hasibiqbal207
hasibiqbal207 committed Sep 3, 2018
1 parent 347c943 commit 3c83094
Show file tree
Hide file tree
Showing 8 changed files with 131 additions and 116 deletions.
2 changes: 1 addition & 1 deletion MainWindow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def __init__(self):

self.setWindowTitle(WINDOW_TITLE)
self.main_image_label.setScaledContents(True)
self.gaze_image_label.setScaledContents(True)
self.left_gaze_label.setScaledContents(True)
# self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
# self.b = QtGui.QPushButton("exit", self, clicked=self.close)

Expand Down
10 changes: 5 additions & 5 deletions UI/MainWindow.ui
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@
</layout>
</item>
<item>
<widget class="QLabel" name="image_info_textlabel">
<widget class="QLabel" name="left_gaze_label">
<property name="minimumSize">
<size>
<width>270</width>
Expand All @@ -224,29 +224,29 @@
</font>
</property>
<property name="text">
<string>Not initialized yet</string>
<string>Eye Gaze Left</string>
</property>
<property name="scaledContents">
<bool>false</bool>
</property>
<property name="alignment">
<set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
<set>Qt::AlignCenter</set>
</property>
<property name="margin">
<number>19</number>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="gaze_image_label">
<widget class="QLabel" name="right_gaze_label">
<property name="minimumSize">
<size>
<width>30</width>
<height>70</height>
</size>
</property>
<property name="text">
<string>Eye Gaze not found</string>
<string>Eye Gaze Right</string>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
Expand Down
171 changes: 89 additions & 82 deletions inputs/Controller.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from PyQt5.QtGui import QImage, QPixmap

import MainWindow
from inputs.BlinkDetector import BlinkDetector
from inputs.FaceDetector import FaceDetector
from inputs.GazeDetectorCnn import GazeDetector
from inputs.Speech import Speech
Expand All @@ -19,7 +18,6 @@
class Controller:
def __init__(self, main_window, giveOutput):
self.gazeImg = None
self.blink_detector = BlinkDetector()
self.face_detector = FaceDetector()
self.gaze_detector = GazeDetector()
# self.speech_detector = Speech()
Expand All @@ -28,91 +26,100 @@ def __init__(self, main_window, giveOutput):
self.cap = cv2.VideoCapture(0)

def getInput(self):
dicBlink = None
dicGaze = None
dicHead = None
label = ""
self.drawImg = None
img = None

# Blink
if True:
_, img = self.cap.read()
dicBlink = self.blink_detector.processImage(img, self.main_window.eyeThreshold.value())

drawImg = dicBlink["img"]

label = label + \
"Blink Detector : " + "\n" \
"\t" + "both : " + str(dicBlink["both"]) + "\n" \
"\t" + "left : " + str(
dicBlink["left"]) + "\n" \
"\t" + "right : " + str(dicBlink["right"]) + "\n" \
"\t" + "leftEAR : " + str(
dicBlink["leftEAR"]) + "\n" \
"\t" + "rightEAR :" + str(dicBlink["rightEAR"]) + "\n" \
"\t" + "bothTotal : " + str(
dicBlink["bothTotal"]) + "\n" \
"\t" + "leftTotal : " + str(dicBlink["leftTotal"]) + "\n" \
"\t" + "rightTotal : " + str(
dicBlink["rightTotal"]) + "\n" \
\
# Head
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
dicHead = self.face_detector.processImage(img, drawImg)

label = label + \
"Head Detector : " + "\n" \
"\t" + "direction : " + str(dicHead["direction"]) + "\n"

# GAZE
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.EYE_HELP \
and self.main_window.current_mode in [MainWindow.MODE.CHAIR, MainWindow.MODE.KEYBOARD]:

dicGaze = self.gaze_detector.processImage(img)

label = label + \
"Gaze Detector : " + "\n" \
"\t" + "Gaze : " + str(dicGaze["direction"]) + "\n"

if dicGaze["img"] is not None:
self.gazeImg = dicGaze["img"]
self.gazeImg = toQImage(self.gazeImg)
self.gazeImg = self.gazeImg.rgbSwapped()

if self.gazeImg is not None:
self.main_window.gaze_image_label.setPixmap(QPixmap.fromImage(self.gazeImg))
else:
self.main_window.gaze_image_label.setText("None")

outImage = toQImage(drawImg)
# if True:
# _, img = self.cap.read()
# dicBlink = self.blink_detector.processImage(img, self.main_window.eyeThreshold.value())
#
# drawImg = dicBlink["img"]
#
# label = label + \
# "Blink Detector : " + "\n" \
# "\t" + "both : " + str(dicBlink["both"]) + "\n" \
# "\t" + "left : " + str(
# dicBlink["left"]) + "\n" \
# "\t" + "right : " + str(dicBlink["right"]) + "\n" \
# "\t" + "leftEAR : " + str(
# dicBlink["leftEAR"]) + "\n" \
# "\t" + "rightEAR :" + str(dicBlink["rightEAR"]) + "\n" \
# "\t" + "bothTotal : " + str(
# dicBlink["bothTotal"]) + "\n" \
# "\t" + "leftTotal : " + str(dicBlink["leftTotal"]) + "\n" \
# "\t" + "rightTotal : " + str(
# dicBlink["rightTotal"]) + "\n" \
# \
# # Head
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
# dicHead = self.face_detector.processImage(img, drawImg)
#
# label = label + \
# "Head Detector : " + "\n" \
# "\t" + "direction : " + str(dicHead["direction"]) + "\n"
#
# # GAZE
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.EYE_HELP \
# and self.main_window.current_mode in [MainWindow.MODE.CHAIR, MainWindow.MODE.KEYBOARD]:
#
# dicGaze = self.gaze_detector.processImage(img)
#
# label = label + \
# "Gaze Detector : " + "\n" \
# "\t" + "Gaze : " + str(dicGaze["direction"]) + "\n"
#
# if dicGaze["img"] is not None:
# self.gazeImg = dicGaze["img"]
# self.gazeImg = toQImage(self.gazeImg)
# self.gazeImg = self.gazeImg.rgbSwapped()
#
# if self.gazeImg is not None:
# self.main_window.gaze_image_label.setPixmap(QPixmap.fromImage(self.gazeImg))
# else:
# self.main_window.gaze_image_label.setText("None")


# Blink and Gaze
_, img = self.cap.read()
dicGaze = self.gaze_detector.processImage(img)

outImage = toQImage(dicGaze["img"])
outImage = outImage.rgbSwapped()
self.main_window.main_image_label.setPixmap(QPixmap.fromImage(outImage))
self.main_window.image_info_textlabel.setText(label)

if dicBlink is not None:
if dicBlink["left"]:
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
self.face_detector.initPos(dicHead["face"])
return
self.giveOutput("blinkleft")
return
elif dicBlink["right"]:
self.giveOutput("blinkright")
return
elif dicBlink["both"]:
self.giveOutput("blinkboth")
return

if dicHead is not None:
if dicHead["direction"] != "not initialized":
self.giveOutput(dicHead["direction"])
return

if dicGaze is not None:
if dicGaze["direction"] != "not initialized":
self.giveOutput(dicGaze["direction"])
return

if dicGaze["gazeleft"] is not None:
outImage = toQImage(dicGaze["gazeleft"])
outImage = outImage.rgbSwapped()
self.main_window.left_gaze_label.setPixmap(QPixmap.fromImage(outImage))

if dicGaze["gazeright"] is not None:
outImage = toQImage(dicGaze["gazeright"])
outImage = outImage.rgbSwapped()
self.main_window.right_gaze_label.setPixmap(QPixmap.fromImage(outImage))

# if dicBlink is not None:
# if dicBlink["left"]:
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
# self.face_detector.initPos(dicHead["face"])
# return
# self.giveOutput("blinkleft")
# return
# elif dicBlink["right"]:
# self.giveOutput("blinkright")
# return
# elif dicBlink["both"]:
# self.giveOutput("blinkboth")
# return

# if dicHead is not None:
# if dicHead["direction"] != "not initialized":
# self.giveOutput(dicHead["direction"])
# return
#
# if dicGaze is not None:
# if dicGaze["direction"] != "not initialized":
# self.giveOutput(dicGaze["direction"])
# return


def toQImage(raw_img):
Expand Down
64 changes: 36 additions & 28 deletions inputs/GazeDetectorCnn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ def __init__(self):
self.predictor = dlib.shape_predictor(PREDICTOR_PATH)
# cap = cv2.VideoCapture(0)


def processImage(self, image):

ret = {
"direction": None,
"img": None,
"gaze": None
}

# a = datetime.datetime.now()
Expand Down Expand Up @@ -73,7 +73,6 @@ def processImage(self, image):
roi2 = image[y:y + h, x:x + w]
roi2 = imutils.resize(roi2, width=250, height=250, inter=cv2.INTER_CUBIC)


if roi1 is not None:
cv2.imwrite('temp_right.jpg', roi1)
img = cv2.imread('temp_right.jpg')
Expand All @@ -82,6 +81,7 @@ def processImage(self, image):
classes_r81 = self.model_right.predict_classes(img)
classes_r82 = self.model_right.predict(img)
class_no = classes_r81.item(0)
ret["gazeleft"] = roi1

if roi2 is not None:
cv2.imwrite('temp_left.jpg', roi2)
Expand All @@ -91,55 +91,63 @@ def processImage(self, image):
classes_l81 = self.model_left.predict(img)
classes_l82 = self.model_left.predict(img)
class_no = classes_l81.item(0)
ret["gazeright"] = roi2

ret["img"] = roi1
ret["img"] = image

if roi1 is not None or roi2 is not None:
self.percent(classes_l81, classes_r82, roi1, ret)
self.percent(classes_l81, classes_r82, ret)

return ret

def percent(self, values_left, values_right, image, ret):
def percent(self, values_left, values_right, ret):
x_l = values_left.item(0)
y_l = values_left.item(1)
z_l = values_left.item(2)
b_l = values_left.item(3)

x_r = values_right.item(0)
y_r = values_right.item(1)
z_r = values_right.item(2)
b_r = values_right.item(3)

total_l = x_l + y_l + z_l
total_r = x_r + y_r + z_r
total_l = b_l + x_l + y_l + z_l
total_r = b_r + x_r + y_r + z_r

x_l = float("{0:.2f}".format(x_l / total_l))
y_l = float("{0:.2f}".format(y_l / total_l))
z_l = float("{0:.2f}".format(z_l / total_l))
b_l = float("{0:.2f}".format(b_l / total_l))

x_r = float("{0:.2f}".format(x_r / total_r))
y_r = float("{0:.2f}".format(y_r / total_r))
z_r = float("{0:.2f}".format(z_r / total_r))
b_r = float("{0:.2f}".format(b_r / total_r))

x = (x_l + x_r) / 2
y = (y_l + y_r) / 2
z = (z_l + z_r) / 2
# print("Prob of Class 0 is : ", x, "\nProb of Class 1 is : ", y, "\nProb of Class 2 is : ", z)

if (x >= 0.80):
cv2.putText(image, "LEFT", (15, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 155, thickness=2)
ret["direction"] = "gazeleft"
elif (z >= 0.80):
cv2.putText(image, "RIGHT", (25, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 130, thickness=2)
ret["direction"] = "gazeright"
elif (y >= 0.80):
cv2.putText(image, "Middle", (40, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 255, thickness=2)
ret["direction"] = "gazecenter"

def reset(self):
self.init = [0, 0]
self.coordinate = [0, 0]
self.dir_c = 0
self.dir_l = 0
self.dir_r = 0

def closeAll(self):
cv2.destroyAllWindows()
b = (b_l + b_r) / 2
print("Prob of Class 0 is : ", x, "\nProb of Class 1 is : ", y, "\nProb of Class 2 is : ", z,
"\nProb of Class 3 is : ", b)

if x >= 0.80:
cv2.putText(ret["img"], "Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 155, thickness=3)
ret["blink"] = "both"
elif x_l >= 0.80 and x_r <= 0.50:
cv2.putText(ret["img"], "LEFT Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
ret["blink"] = "left"
elif x_l <= 0.50 and x_r >= 0.80:
cv2.putText(ret["img"], "RIGHT Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
ret["blink"] = "right"
else:
ret["blink"] = "none"
if y >= 0.80:
cv2.putText(ret["img"], "LEFT", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
ret["gazedirection"] = "left"
elif z >= 0.80:
cv2.putText(ret["img"], "Middle", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 130, thickness=3)
ret["gazedirection"] = "center"
elif b >= 0.80:
cv2.putText(ret["img"], "RIGHT", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
ret["gazedirection"] = "right"
Binary file modified inputs/lefteyemodel.h5
View file
Binary file not shown.
Binary file modified inputs/righteyemodel.h5
View file
Binary file not shown.
Binary file modified temp_left.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified temp_right.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 3c83094

Please sign in to comment.