Merge pull request #141 from Azure-Samples/yadavm_factoryai

Yadavm factoryai

Author: initmahesh

factory-ai-vision/EdgeSolution/modules/InferenceModule/main.py

@@ -48,6 +48,30 @@ def is_inside_aoi(x1, y1, x2, y2, aoi_info):
             return True
     return False
 
+def parse_bbox(prediction, width, height):
+    x1 = int(prediction['boundingBox']['left'] * width)
+    y1 = int(prediction['boundingBox']['top'] * height)
+    x2 = x1 + int(prediction['boundingBox']['width'] * width)
+    y2 = y1 + int(prediction['boundingBox']['height'] * height)
+    return (x1, y1), (x2, y2)
+
+
+def draw_confidence_level(img, prediction):
+    height, width = img.shape[0], img.shape[1]
+
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    font_scale = 1
+    thickness = 3
+
+    prob_str = str(int(prediction['probability']*1000)/10)
+    prob_str = ' (' + prob_str + '%)'
+
+    (x1, y1), (x2, y2) = parse_bbox(prediction, width, height)
+
+    img = cv2.putText(img, prediction['tagName']+prob_str, (x1+10, y1+30), font, font_scale, (0, 0, 255), thickness)
+
+    return img
+
 
 class ONNXRuntimeModelDeploy(ObjectDetection):
     """Object Detection class for ONNX Runtime
@@ -225,20 +249,8 @@ def run(self):
                     detection = DETECTION_TYPE_NOTHING
                     if True:
                         send_counter += 1
-                        # Modify here to change the threshold (deprecated)
-                        #if send_counter == 200:
-                        #    if iot:
-                        #        iot.send_message_to_output(json.dumps(self.last_prediction), 'metrics')
-                        #    else:
-                        #        print('[METRICS]', json.dumps(self.last_prediction))
-                        #    send_counter = 0
                         if self.iothub_is_send:
                             if self.iothub_last_send_time + self.iothub_interval < time.time():
-                                #print('1', self.iothub_last_send_time)
-                                #print('2', self.iothub_interval)
-                                #print('3', self.iothub_last_send_time + self.iothub_interval)
-                                #print('4', time.time())
-                                #print('wew')
                                 predictions_to_send = []
                                 for prediction in self.last_prediction:
                                     _tag = prediction['tagName']
@@ -262,40 +274,38 @@ def run(self):
                                         pass
                                     self.iothub_last_send_time = time.time()
 
-
-
-
                         for prediction in self.last_prediction:
 
                             tag = prediction['tagName']
-                            if tag not in self.parts: continue
-
-                            if self.last_upload_time + UPLOAD_INTERVAL < time.time():
-                                x1 = int(prediction['boundingBox']['left'] * width)
-                                y1 = int(prediction['boundingBox']['top'] * height)
-                                x2 = x1 + int(prediction['boundingBox']['width'] * width)
-                                y2 = y1 + int(prediction['boundingBox']['height'] * height)
-                                labels = json.dumps([{'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2}])
+                            if tag not in self.parts:
+                                continue
 
-                                if self.has_aoi:
-                                    if not is_inside_aoi(x1, y1, x2, y2, self.aoi_info): continue
+                            (x1, y1) , (x2, y2) = parse_bbox(prediction, width, height)
+                            if self.has_aoi:
+                                if not is_inside_aoi(x1, y1, x2, y2, self.aoi_info): continue
 
-                                if prediction['probability'] > self.confidence_max:
+                            if detection != DETECTION_TYPE_SUCCESS:
+                                if prediction['probability'] >= self.threshold:
                                     detection = DETECTION_TYPE_SUCCESS
-
-                                elif self.confidence_min <= prediction['probability'] <= self.confidence_max:
-
-                                    if detection != DETECTION_TYPE_SUCCESS: detection = DETECTION_TYPE_UNIDENTIFIED
+                                else:
+                                    detection = DETECTION_TYPE_UNIDENTIFIED
 
 
+                            if self.last_upload_time + UPLOAD_INTERVAL < time.time():
+                                if self.confidence_min <= prediction['probability'] <= self.confidence_max:
                                     if self.is_upload_image:
-                                        if tag in onnx.current_uploaded_images and self.current_uploaded_images[tag] >= onnx.max_images:
-                                            pass
-                                        else:
-                                            self.current_uploaded_images[tag] = self.current_uploaded_images.get(tag, 0) + 1
-                                            #print(tag, onnx.current_uploaded_images[tag], j) 
+                                        #if tag in onnx.current_uploaded_images and self.current_uploaded_images[tag] >= onnx.max_images:
+                                        #if tag in onnx.current_uploaded_images:
+                                            # No limit for the max_images in inference module now, the logic is moved to webmodule
+                                        #    pass
+                                        #else:
+                                        if True:
+
+                                            labels = json.dumps([{'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2}])
+                                            print('[INFO] Sending Image to relabeling', tag, onnx.current_uploaded_images.get(tag, 0), labels)
+                                            #self.current_uploaded_images[tag] = self.current_uploaded_images.get(tag, 0) + 1
                                             self.last_upload_time = time.time()
-                                            print('[INFO] Sending Image to relabeling', tag, onnx.current_uploaded_images[tag], labels)
+
                                             jpg = cv2.imencode('.jpg', img)[1].tobytes()
                                             try:
                                                 requests.post('http://'+web_module_url()+'/api/relabel', data={
@@ -324,15 +334,14 @@ def run(self):
                                 self.detection_unidentified_num += 1
                             elif detection == DETECTION_TYPE_SUCCESS:
                                 self.detection_success_num += 1
-                        
                         else:
                             self.detections.append(detection)
                             if detection == DETECTION_TYPE_UNIDENTIFIED:
                                 self.detection_unidentified_num += 1
                             elif detection == DETECTION_TYPE_SUCCESS:
                                 self.detection_success_num += 1
                             self.detection_total += 1
-                        
+
                     self.lock.release()
                     #print(detection)
                 else:
@@ -389,7 +398,7 @@ def metrics():
 
 @app.route('/update_retrain_parameters')
 def update_retrain_parameters():
-    
+
     confidence_min = request.args.get('confidence_min')
     if not confidence_min: return 'missing confidence_min'
 
@@ -485,15 +494,10 @@ def update_parts():
 
     return 'ok'
 
-@app.route('/update_threshold')
-def update_threshold():
-    #threshold = float(request.args.get('threshold'))
-
-    #print('[INFO] update theshold to', threshold)
-
-    #onnx.threshold = threshold
-    print('[WARNING] is depreciated')
-    return 'ok'
+#@app.route('/update_threshold')
+#def update_threshold():
+#    print('[WARNING] is depreciated')
+#    return 'ok'
 
 @app.route('/update_iothub_parameters')
 def update_iothub_parameters():
@@ -527,6 +531,13 @@ def update_prob_threshold():
     print('[INFO] updaing prob_threshold to')
     print('  prob_threshold:', prob_threshold)
 
+    onnx.lock.acquire()
+    onnx.detection_success_num = 0
+    onnx.detection_unidentified_num = 0
+    onnx.detection_total = 0
+    onnx.detections = []
+    onnx.lock.release()
+
     return 'ok'
 
 @app.route('/video_feed')
@@ -539,9 +550,6 @@ def _gen():
             if inference:
                 height, width = img.shape[0], img.shape[1]
                 predictions = onnx.last_prediction
-                font = cv2.FONT_HERSHEY_SIMPLEX
-                font_scale = 1
-                thickness = 3
                 for prediction in predictions:
                     tag = prediction['tagName']
                     if tag not in onnx.parts: continue
@@ -550,33 +558,22 @@ def _gen():
                         for aoi_area in onnx.aoi_info:
                             img = cv2.rectangle(img, (int(aoi_area['x1']), int(aoi_area['y1'])), (int(aoi_area['x2']), int(aoi_area['y2'])), (0, 255, 255), 2)
 
-                    #if prediction['probability'] > onnx.confidence_max:
-                    #print(prediction)
                     if prediction['probability'] > onnx.threshold:
-                        x1 = int(prediction['boundingBox']['left'] * width)
-                        y1 = int(prediction['boundingBox']['top'] * height)
-                        x2 = x1 + int(prediction['boundingBox']['width'] * width)
-                        y2 = y1 + int(prediction['boundingBox']['height'] * height)
+                        (x1, y1), (x2, y2) = parse_bbox(prediction, width, height)
                         if onnx.has_aoi:
                             if not is_inside_aoi(x1, y1, x2, y2, onnx.aoi_info): continue
 
                         img = cv2.rectangle(img, (x1, y1), (x2, y2), (0, 0, 255), 2)
-                        prob_str = str(int(prediction['probability']*1000)/10)
-                        prob_str = ' (' + prob_str + '%)'
-                        img = cv2.putText(img, prediction['tagName']+prob_str, (x1+10, y1+30), font, font_scale, (0, 0, 255), thickness)
+                        img = draw_confidence_level(img, prediction)
 
 
-            time.sleep(0.03)
+            time.sleep(0.02)
             yield (b'--frame\r\n'
                        b'Content-Type: image/jpeg\r\n\r\n' + cv2.imencode('.jpg', img)[1].tobytes() + b'\r\n')
 
     return Response(_gen(),
             mimetype='multipart/x-mixed-replace; boundary=frame')
 
-# for debugging
-#onnx.update_cam(cam_type='rtsp', cam_source='sample_video/video.mp4', has_aoi=True, aoi_info={'x1': 100, 'x2': 1000, 'y1': 100, 'y2': 500})
-#requests.get('http://localhost:5000/update_cam', params={'cam_type':'rtsp', 'cam_source':'0', 'aoi':'{\"useAOI\":true,\"AOIs\":[{\"x1\":100,\"y1\":216.36363636363637,\"x2\":3314.909090909091,\"y2\":1762.181818181818}]}'})
-
 
 def main():
 

factory-ai-vision/EdgeSolution/modules/InferenceModule/module.json

@@ -4,7 +4,7 @@
   "image": {
     "repository": "${CONTAINER_REGISTRY_NAME}/intelligentedge/inferencemodule",
     "tag": {
-      "version": "0.1.70",
+      "version": "0.1.74",
       "platforms": {
         "cpuamd64": "./Dockerfile.cpuamd64",
         "gpuamd64": "./Dockerfile.gpuamd64",

factory-ai-vision/EdgeSolution/modules/WebModule/backend/README.md

@@ -1,4 +1,4 @@
-# Camera App
+# Vision On Edge Web Module
 
 ## How to run
 
@@ -35,12 +35,11 @@ pylint
 Go to project directory (same level with manage.py)
 
 ```bash
-python manage.py test cameras.tests
+python manage.py test
 ```
 
 ## Test coverage
 
 ```bash
-coverage run --source='cameras' manage.py test cameras.tests
+coverage run --source='vision_on_edge' manage.py test vision_on_edge.tests
 ```
-

factory-ai-vision/EdgeSolution/modules/WebModule/backend/configs/api.py

@@ -0,0 +1,17 @@
+"""
+Websocket router
+"""
+from channels.routing import ProtocolTypeRouter, URLRouter
+from channels.security.websocket import AllowedHostsOriginValidator
+from django.conf.urls import url
+
+from vision_on_edge.notifications.consumers import NotificationConsumer
+
+application = ProtocolTypeRouter({
+    # (http->django views is added by default)
+    'websocket':
+        AllowedHostsOriginValidator(
+            URLRouter([
+                url(r"api/notifications", NotificationConsumer),
+            ]))
+})

factory-ai-vision/EdgeSolution/modules/WebModule/backend/configs/routing.py

@@ -44,11 +44,16 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'corsheaders',
+    'channels',
+    'vision_on_edge.azure_parts',
+    'vision_on_edge.images',
+    'vision_on_edge.streams',
     'vision_on_edge.azure_settings',
     'vision_on_edge.locations',
     'vision_on_edge.cameras',
     'vision_on_edge.image_predictions',
     'vision_on_edge.azure_training',
+    'vision_on_edge.notifications',
     'rest_framework',
     'drf_yasg',
 ]
@@ -99,8 +104,7 @@
 
 CORS_ORIGIN_ALLOW_ALL = True
 
-# WSGI_APPLICATION = 'vision_on_edge.wsgi.application'
-ASGI_APPLICATION = 'vision_on_edge.asgi.application'
+ASGI_APPLICATION = 'configs.routing.application'
 
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
@@ -163,6 +167,12 @@
 ICON_URL = '/icons/'
 ICON_ROOT = os.path.join(UI_DIR, 'icons')
 
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels.layers.InMemoryChannelLayer"
+    }
+}
+
 IOT_HUB_CONNECTION_STRING = config.IOT_HUB_CONNECTION_STRING
 DEVICE_ID = config.DEVICE_ID
 MODULE_ID = config.MODULE_ID

factory-ai-vision/EdgeSolution/modules/WebModule/backend/configs/settings.py

@@ -20,14 +20,18 @@
 from django.urls import path
 from rest_framework import routers
 
-from vision_on_edge.azure_settings.api import views as setting_views
+from vision_on_edge.azure_parts.api import views as azure_part_views
+from vision_on_edge.azure_settings.api import views as azure_setting_views
 from vision_on_edge.azure_training.api import views as azure_training_views
 from vision_on_edge.cameras.api import util_views as camera_util_views
 from vision_on_edge.cameras.api import views
 from vision_on_edge.image_predictions.api import \
     views as image_prediction_views
+from vision_on_edge.images.api import views as image_views
 from vision_on_edge.locations.api import views as location_views
-
+from vision_on_edge.relabeling.api import views as relabel_views
+from vision_on_edge.streams.api import views as stream_views
+from vision_on_edge.notifications.api import views as notifications_views
 from . import views as site_views
 
 
@@ -41,34 +45,36 @@ def __init__(self):
 
 #router = ters.DefaultRouter(trailing_slash=False)
 router = OptionalSlashRouter()
-router.register('settings', setting_views.SettingViewSet)
+router.register('settings', azure_setting_views.SettingViewSet)
 router.register('cameras', views.CameraViewSet)
-router.register('parts', views.PartViewSet)
-router.register('images', views.ImageViewSet)
+router.register('parts', azure_part_views.PartViewSet)
 router.register('locations', location_views.LocationViewSet)
-router.register('annotations', views.AnnotationViewSet)
 router.register('image_predictions',
                 image_prediction_views.ImagePredictionViewSet)
 router.register('projects', azure_training_views.ProjectViewSet)
 router.register('train', azure_training_views.TrainViewSet)
 router.register('tasks', azure_training_views.TaskViewSet)
+router.register('images', image_views.ImageViewSet)
+router.register('notifications', notifications_views.NotificationViewSet)
+router.register('images', image_views.ImageViewSet)
 
 urlpatterns = \
     static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + \
     static(settings.ICON_URL, document_root=settings.ICON_ROOT) + \
     [
         url('^api/', include(router.urls)),
-        url('api/streams/connect', views.connect_stream),
-        path('api/streams/<int:stream_id>/disconnect', views.disconnect_stream),
-        path('api/streams/<int:stream_id>/video_feed', views.video_feed),
-        path('api/streams/<int:stream_id>/capture', views.capture),
+        url('api/streams/connect', stream_views.connect_stream),
+        path('api/streams/<int:stream_id>/disconnect',
+             stream_views.disconnect_stream),
+        path('api/streams/<int:stream_id>/video_feed', stream_views.video_feed),
+        path('api/streams/<int:stream_id>/capture', stream_views.capture),
         path('api/projects/<int:project_id>/train', azure_training_views.train),
         path('api/projects/<int:project_id>/export',
              azure_training_views.export),
         path('api/projects/<int:project_id>/train_performance',
              azure_training_views.train_performance),
         path('api/projects/<int:project_id>/inference_video_feed',
-             views.inference_video_feed),
+             stream_views.inference_video_feed),
         path('api/projects/<int:project_id>/pull_cv_project',
              azure_training_views.pull_cv_project),
         path('api/projects/<int:project_id>/update_prob_threshold',
@@ -78,8 +84,8 @@ def __init__(self):
         path('api/projects/<int:project_id>/reset_camera',
              azure_training_views.project_reset_camera),
         path('api/projects/null/export', azure_training_views.export_null),
-        path('api/relabel', views.upload_relabel_image),
-        path('api/relabel/update', views.relabel_update),
+        path('api/relabel', relabel_views.upload_relabel_image),
+        path('api/relabel/update', relabel_views.relabel_update),
         path('api/appinsight/key', views.instrumentation_key),
         path('api/camera_utils/verify_rtsp',
              camera_util_views.verify_rtsp),