GFBDetect.py

#By Adrian at PyImageSearch and Rocky43007
#Copyright Rocky43007 2020

# USAGE
# python GFBDetect.py --conf config/config.json

# import the necessary package
# imutils - Used to make using OpenCV easier.
import imutils
# The next 2 are required for the Object detection to work, using files 
# provided by Intel.
from openvino.inference_engine import IENetwork
from openvino.inference_engine import IEPlugin
# These files from Intel help with using TinyYOLO for the object detection.
from intel.yoloparams import TinyYOLOV3Params
from intel.tinyyolo import TinyYOLOv3
# Provided by PyImageSearch.
from imutils.video import VideoStream
from pyimagesearch.utils import Conf
from imutils.video import FPS
# Used to allow Arrays and complex mathematics do be done by Python.
import numpy as np
# The argparse module makes it easy to write user-friendly command-line interfaces.
import argparse
# Time module, nothing fancy.
import time
# Required to get OpenCV to work using Python.
import cv2
# Used for communication between Code and Operating System.
import os
# Used to better utilise system resources.
import subprocess
# Required to get Text to Speech working.
from gtts import gTTS
# Required for audio to work properly.
import pyaudio
# Required to get data from Ultrasonic sensor (The sensor which measures distance).
import RPi.GPIO as GPIO
# Required to find age. Look at AgeFinder.py for more details.
from datetime import date
# Required to get data saved in text from 'Birthday.native'.
import pickle
# Used to allow caching to make the code run faster.
import functools

# Requirements to get the Ultrasonic sensor working. Basically just says 'Ok, you'll get information
# From this pin and this pin on the raspberry pi'.
#set GPIO Mode
GPIO.setmode(GPIO.BCM)
 
#set GPIO Pins
GPIO_TRIGGER = 23
GPIO_ECHO = 24
 
#set GPIO direction (IN / OUT)
GPIO.setup(GPIO_TRIGGER, GPIO.OUT)
GPIO.setup(GPIO_ECHO, GPIO.IN)

# Look at AgeFinder.py for indepth explaination.
with open('birthday.native', 'rb') as f:
    x = pickle.load(f)
    a = x.strip("00:00:00")
    d = a.split("-")

print ("Original list is : " + str(d)) 

for i in range(0, len(d)): 
    d[i] = int(d[i]) 

  
print ("Modified list is : " + str(d)) 
year, month, day = d
print (year, month, day)

@functools.lru_cache(maxsize=128)
def calculateAge(born): 
    today = date.today() 
    try:  
        birthday = born.replace(year = today.year) 
  
    # raised when birth date is February 29 
    # and the current year is not a leap year 
    except ValueError:  
        birthday = born.replace(year = today.year, 
                  month = born.month + 1, day = 1) 
  
    if birthday > today: 
        return today.year - born.year - 1
    else: 
        return today.year - born.year 
          
# Driver code 
age = calculateAge(date(year, month, day))
print (age)

# Main detection code.
@functools.lru_cache(maxsize=128)
def Detect():
    # construct the argument parser and parse the arguments 
    # Basically the code forces the code to use a specific requirement for it to work. 
    # It requires the config file for the data realated to accuracy,
    ap = argparse.ArgumentParser()
    ap.add_argument("-c", "--conf", required=True,
        help="Path to the input configuration file")
    # Extra requirement if you want the code to analyse a video instead of it working real time.
    ap.add_argument("-i", "--input", help="path to the input video file")
    args = vars(ap.parse_args())
    
    # load the configuration file
    conf = Conf(args["conf"])
    
    # load the COCO class labels our YOLO model was trained on and
    # initialize a list of colors to represent each possible class
    # label
    # COCO is the object list for the code. It's what alows the code to recognise if
    # the user is looking at a chair or a human.
    LABELS = open(conf["labels_path"]).read().strip().split("\n")
    np.random.seed(42)
    COLORS = np.random.uniform(0, 255, size=(len(LABELS), 3))

    # initialize the plugin in for specified device
    # This is basically initializing the NCS2 which is required for the extra
    # visual processing power.
    plugin = IEPlugin(device="MYRIAD")
    
    # read the IR generated by the Model Optimizer (.xml and .bin files)
    print("[INFO] loading models...")
    net = IENetwork(model=conf["xml_path"], weights=conf["bin_path"])
    
    # prepare inputs
    print("[INFO] preparing inputs...")
    inputBlob = next(iter(net.inputs))
    
    # set the default batch size as 1 and get the number of input blobs,
    # number of channels, the height, and width of the input blob
    net.batch_size = 1
    (n, c, h, w) = net.inputs[inputBlob].shape
    
    # if a video path was not supplied, grab a reference to the webcam
    if args["input"] is None:
        print("[INFO] starting video stream...")
        # vs = VideoStream(src=0).start()
        vs = VideoStream(usePiCamera=True).start()
        time.sleep(2.0)
        
    # otherwise, grab a reference to the video file
    else:
        print("[INFO] opening video file...")
        vs = cv2.VideoCapture(os.path.abspath(args["input"]))
    
    # loading model to the plugin and start the frames per second
    # throughput estimator
    print("[INFO] loading model to the plugin...")
    execNet = plugin.load(network=net, num_requests=1)
    fps = FPS().start()
    
    # loop over the frames from the video stream
    while True:
        # grab the next frame and handle if we are reading from either
        # VideoCapture or VideoStream
        orig = vs.read()
        orig = orig[1] if args["input"] is not None else orig
    
        # if we are viewing a video and we did not grab a frame then we
        # have reached the end of the video
        if args["input"] is not None and orig is None:
            break

        # resize original frame to have a maximum width of 500 pixel and
        # input_frame to network size
        orig = imutils.resize(orig, width=500)
        frame = cv2.resize(orig, (w, h))
    
        # change data layout from HxWxC to CxHxW
        frame = frame.transpose((2, 0, 1))
        frame = frame.reshape((n, c, h, w))
    
        # start inference and initialize list to collect object detection
        # results
        output = execNet.infer({inputBlob: frame})
        objects = []
    
        # loop over the output items
        for (layerName, outBlob) in output.items():
            # create a new object which contains the required tinyYOLOv3
            # parameters
            layerParams = TinyYOLOV3Params(net.layers[layerName].params,
                outBlob.shape[2])

            # parse the output region
            objects += TinyYOLOv3.parse_yolo_region(outBlob,
                frame.shape[2:], orig.shape[:-1], layerParams,
                conf["prob_threshold"])
            
    
        # loop over each of the objects
        for i in range(len(objects)):
            # check if the confidence of the detected object is zero, if
            # it is, then skip this iteration, indicating that the object
            # should be ignored
            if objects[i]["confidence"] == 0:
                continue

            # loop over remaining objects
            for j in range(i + 1, len(objects)):
                # check if the IoU of both the objects exceeds a
                # threshold, if it does, then set the confidence of that
                # object to zero
                if TinyYOLOv3.intersection_over_union(objects[i],
                    objects[j]) > conf["iou_threshold"]:
                    objects[j]["confidence"] = 0
    
        # filter objects by using the probability threshold -- if a an
        # object is below the threshold, ignore it
        objects = [obj for obj in objects if obj['confidence'] >= \
            conf["prob_threshold"]]
        
    
        # store the height and width of the original frame
        (endY, endX) = orig.shape[:-1]

        # loop through all the remaining objects
        for obj in objects:
            # validate the bounding box of the detected object, ensuring
            # we don't have any invalid bounding boxes
            if obj["xmax"] > endX or obj["ymax"] > endY or obj["xmin"] \
                < 0 or obj["ymin"] < 0:
                continue
    
            # build a label consisting of the predicted class and
            # associated probability
            label = "{}: {:.2f}%".format(LABELS[obj["class_id"]],
                obj["confidence"] * 100)
            
            print (label)
            
            # This part of code helps us find the number of steps from the user to the object.
            @functools.lru_cache(maxsize=128)
            def distance():
                # Using the age of the user and the Ultrasonic sensor, we can find the steps required.
                age = calculateAge(date(year, month, day))
                # set Trigger to HIGH
                GPIO.output(GPIO_TRIGGER, True)

                # set Trigger after 0.01ms to LOW
                time.sleep(0.0001)
                GPIO.output(GPIO_TRIGGER, False)
                StartTime = time.time()
                StopTime = time.time()

                # save StartTime
                while GPIO.input(GPIO_ECHO) == 0:
                    StartTime = time.time()
                # save time of arrival
                while GPIO.input(GPIO_ECHO) == 1:
                    StopTime = time.time()

                # time difference between start and arrival
                TimeElapsed = StopTime - StartTime
                # multiply with the sonic speed (34300 cm/s)
                # and divide by 2, because there and back
                distance = (TimeElapsed * 34300) / 2

                return distance

            # This piece of code helps with finding the distance using meters, as the code 
            # above saves it as centimeters.
            dist = distance()
            # Converts cm to meters.
            meter = dist / 100.0;
            # States it for debuging for errors.
            measure = " and the measured distance is %.1f m" % meter
            print (measure)
            
            # Maths required to find age.
            # This is a bunch of if statements checking what the age is, and then 
            # giving the corresponding pace of the user.
            # Checks if the age is in which range.
            if age in range(20,29):
                # Prints the pace for debuging for errors.
                print ('Steps = 1.35 m/s')
                # Gives us the number of steps by taking the number of meters and dividing
                # by the pace set. 
                # The code rounds the steps to give a whole number, as you can't have 
                # 6.5467 steps, which the user won't really understand.
                steps = round(meter/1.35)
                print (steps)
            
            # Same as the previous statement, just different age group and pace.
            elif age in range(30,39):
                print ('Steps = 1.385 m/s')
                steps = round(meter/1.385)
                print (steps)
            # Same as the previous statement, just different age group and pace.
            elif age in range(40,49):
                print ('Steps = 1.41 m/s')
                steps = round(meter/1.41)
                print (steps)
            # Same as the previous statement, just different age group and pace.
            elif age in range(50,59):
                print ('Steps = 1.37 m/s')
                steps = round(meter/1.37)
                print (steps)
            # Same as the previous statement, just different age group and pace.
            elif age in range(60,69):
                print ('Steps = 1.29 m/s')
                steps = round(meter/1.29)
                print (steps)
            # Same as the previous statement, just different age group and pace.
            elif age in range(70,79):
                print ('Steps = 1.195 m/s')
                steps = round(meter/1.195)
                print (steps)
            # Same as the previous statement, just different age group and pace.
            elif age in range(80,89):
                print ('Steps = 0.96m/s')
                steps = round(meter/0.96)
                print (steps)
            
            # This is the statement which will then give us the number of steps in a sentence,
            # while also preseting the sentence for the audio portion.
            stepstxt = " and the object is %s steps away." %steps
            # Prints the text for dev for debugging for errors.
            print (stepstxt)
            
            # Sets the language which the audio will be played in.
            language = 'en'
            # Prints the object with the percentage as well as the steps text for
            # Debugging purposes.
            print (label + stepstxt)
            # This piece of code is what turns the text part of the code into audio as a mp3 file.
            output = gTTS(text=label+stepstxt, lang='en', slow=False)

            # Saves the file as a mp3 file with a name.
            output.save('output.mp3')

            # Uses the operating system to play the audio using the terminal without having to
            # open any apps.
            os.system('mpg123 output.mp3')

            # This code is just more requirements for the realtime viewing window for demonstration
            # and debuging.    
            # calculate the y-coordinate used to write the label on the
            # frame depending on the bounding box coordinate
            y = obj["ymin"] - 15 if obj["ymin"] - 15 > 15 else \
                obj["ymin"] + 15
    
            # draw a bounding box rectangle and label on the frame
            cv2.rectangle(orig, (obj["xmin"], obj["ymin"]), (obj["xmax"],
                obj["ymax"]), COLORS[obj["class_id"]], 2)
            cv2.putText(orig, label, (obj["xmin"], y),
                cv2.FONT_HERSHEY_SIMPLEX, 1, COLORS[obj["class_id"]], 3)
            

        # display the current frame to the screen and record if a user
        # presses a key
        cv2.imshow("TinyYOLOv3", orig)
        key = cv2.waitKey(1) & 0xFF
        
        # if the `q` key was pressed, break from the loop
        if key == ord("q"):
            break

        # update the FPS counter
        fps.update()

    # stop the timer and display FPS information
    fps.stop()
    print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
    
    # stop the video stream and close any open windows
    vs.stop() if args["input"] is None else vs.release()
    cv2.destroyAllWindows()
# This is required to make the code run, as this wouldn't have worked without it all being in a loop of sorts.
# This is basically a command for the code to run when asked. 
Detect()