Deal with edge cases

Isolate was being fidgety about transferring the Status over isolate barriers. Timestamp needs to be kept inside the graph runner and then recovered so that when switching from camera and live image you always get the latest timestamp.
openvinotoolkit · Feb 4, 2025 · 736c6b5 · 736c6b5
1 parent 8525a1c
commit 736c6b5
Show file tree

Hide file tree

Showing 11 changed files with 97 additions and 11 deletions.
diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart
@@ -47,6 +47,20 @@ class OpenVINO {
   late final _freeStatusOrString = _freeStatusOrStringPtr
       .asFunction<void Function(ffi.Pointer<StatusOrString>)>();
 
+  void freeStatusOrInt(
+    ffi.Pointer<StatusOrInt> status,
+  ) {
+    return _freeStatusOrInt(
+      status,
+    );
+  }
+
+  late final _freeStatusOrIntPtr =
+      _lookup<ffi.NativeFunction<ffi.Void Function(ffi.Pointer<StatusOrInt>)>>(
+          'freeStatusOrInt');
+  late final _freeStatusOrInt =
+      _freeStatusOrIntPtr.asFunction<void Function(ffi.Pointer<StatusOrInt>)>();
+
   void freeStatusOrImageInference(
     ffi.Pointer<StatusOrImageInference> status,
   ) {
@@ -843,6 +857,20 @@ class OpenVINO {
       ffi.Pointer<Status> Function(CGraphRunner, int,
           ImageInferenceCallbackFunction, bool, bool, bool)>();
 
+  ffi.Pointer<StatusOrInt> graphRunnerGetTimestamp(
+    CGraphRunner instance,
+  ) {
+    return _graphRunnerGetTimestamp(
+      instance,
+    );
+  }
+
+  late final _graphRunnerGetTimestampPtr = _lookup<
+          ffi.NativeFunction<ffi.Pointer<StatusOrInt> Function(CGraphRunner)>>(
+      'graphRunnerGetTimestamp');
+  late final _graphRunnerGetTimestamp = _graphRunnerGetTimestampPtr
+      .asFunction<ffi.Pointer<StatusOrInt> Function(CGraphRunner)>();
+
   ffi.Pointer<Status> graphRunnerStopCamera(
     CGraphRunner instance,
   ) {

diff --git a/lib/interop/graph_runner.dart b/lib/interop/graph_runner.dart
@@ -36,9 +36,21 @@ class GraphRunner {
     return GraphRunner(result);
   }
 
+  int getTimestamp() {
+    final status = ov.graphRunnerGetTimestamp(instance.ref.value);
+
+    if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
+      throw "GraphRunner::get error: ${status.ref.status} ${status.ref.message.toDartString()}";
+    }
+    final content = status.ref.value;
+    ov.freeStatusOrInt(status);
+    return content;
+  }
+
   Future<String> get() async {
+    int instanceAddress = instance.ref.value.address;
     return await Isolate.run(() {
-      final status = ov.graphRunnerGet(instance.ref.value);
+      final status = ov.graphRunnerGet(Pointer<Void>.fromAddress(instanceAddress));
 
       if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
         throw "GraphRunner::get error: ${status.ref.status} ${status.ref.message.toDartString()}";
@@ -78,12 +90,14 @@ class GraphRunner {
   }
 
   Future<void> queueImage(String nodeName, int timestamp, Uint8List file) async {
+
+    int instanceAddress = instance.ref.value.address;
     await Isolate.run(() {
       final _data =  calloc.allocate<Uint8>(file.lengthInBytes);
       final _bytes = _data.asTypedList(file.lengthInBytes);
       _bytes.setRange(0, file.lengthInBytes, file);
       final nodeNamePtr = nodeName.toNativeUtf8();
-      final status = ov.graphRunnerQueueImage(instance.ref.value, nodeNamePtr, timestamp, _data, file.lengthInBytes);
+      final status = ov.graphRunnerQueueImage(Pointer<Void>.fromAddress(instanceAddress), nodeNamePtr, timestamp, _data, file.lengthInBytes);
       calloc.free(nodeNamePtr);
 
       if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {
@@ -93,9 +107,10 @@ class GraphRunner {
   }
 
   Future<void> queueSerializationOutput(String nodeName, int timestamp, SerializationOutput output) async {
+    int instanceAddress = instance.ref.value.address;
     await Isolate.run(() {
       final nodeNamePtr = nodeName.toNativeUtf8();
-      final status = ov.graphRunnerQueueSerializationOutput(instance.ref.value, nodeNamePtr, timestamp, output.json, output.csv, output.overlay);
+      final status = ov.graphRunnerQueueSerializationOutput(Pointer<Void>.fromAddress(instanceAddress), nodeNamePtr, timestamp, output.json, output.csv, output.overlay);
       calloc.free(nodeNamePtr);
 
       if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) {

diff --git a/lib/pages/computer_vision/live_inference.dart b/lib/pages/computer_vision/live_inference.dart
@@ -38,13 +38,20 @@ class LiveInference extends StatefulWidget {
 
 class _LiveInferenceState extends State<LiveInference> {
   Future<ImageInferenceResult>? inferenceResult;
+  late ImageInferenceProvider inferenceProvider;
   ui.Image? image;
 
   Future<List<CameraDevice>> cameraDevices = CameraDevice.getDevices();
   CameraDevice? cameraDevice;
 
   LiveInferenceMode mode = LiveInferenceMode.image;
 
+  @override
+  void initState() {
+    super.initState();
+    inferenceProvider = Provider.of<ImageInferenceProvider>(context, listen: false);
+  }
+
   void showUploadMenu() async {
     FilePickerResult? result = await FilePicker.platform.pickFiles(type: FileType.image);
 
@@ -54,16 +61,19 @@ class _LiveInferenceState extends State<LiveInference> {
   }
 
   void uploadFile(String path) async {
+    if (mode == LiveInferenceMode.image) {
+      inferenceProvider.closeCamera();
+    }
     setState(() {
+        mode = LiveInferenceMode.image;
+        cameraDevice = null;
         image = null;
         inferenceResult = null;
     });
 
     Uint8List imageData = File(path).readAsBytesSync();
-    final inferenceProvider = Provider.of<ImageInferenceProvider>(context, listen: false);
     final uiImage = await decodeImageFromList(imageData);
     setState(() {
-        mode = LiveInferenceMode.image;
         image = uiImage;
         inferenceResult = inferenceProvider.infer(imageData, SerializationOutput(json: true));
     });
@@ -125,10 +135,10 @@ class _LiveInferenceState extends State<LiveInference> {
                               buttonBuilder: (context, callback) {
                                 return NoOutlineButton(
                                   onPressed: callback,
-                                  child: const Row(
+                                  child: Row(
                                     children: [
-                                      Text("Choose camera"),
-                                      Padding(
+                                      cameraDevice == null ? const Text("Choose camera") : Text("Choose camera: ${cameraDevice!.name}"),
+                                      const Padding(
                                         padding: EdgeInsets.only(left: 8),
                                         child: Icon(FluentIcons.chevron_down, size: 12),
                                       ),

diff --git a/lib/pages/computer_vision/widgets/camera_view.dart b/lib/pages/computer_vision/widgets/camera_view.dart
@@ -1,3 +1,9 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 import 'dart:async';
 import 'dart:convert';
 

diff --git a/lib/providers/image_inference_provider.dart b/lib/providers/image_inference_provider.dart
@@ -17,7 +17,6 @@ class ImageInferenceProvider extends ChangeNotifier {
   Completer<void> loaded = Completer<void>();
   final Project project;
   final String device;
-  int timestamp = 0;
   GraphRunner? _inference;
   GraphRunner? get inference => _inference;
 
@@ -51,9 +50,9 @@ class ImageInferenceProvider extends ChangeNotifier {
   }
 
   Future<ImageInferenceResult> infer(Uint8List file, SerializationOutput output) async {
+    int timestamp = inference!.getTimestamp() + 1;
     _inference!.queueImage("input", timestamp, file);
     _inference!.queueSerializationOutput("serialization_output", timestamp, output);
-    timestamp += 1;
     final result = await _inference!.get();
     return ImageInferenceResult.fromJson(jsonDecode(result));
   }

diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc
@@ -40,6 +40,10 @@ void freeStatusOrString(StatusOrString *status) {
     delete status;
 }
 
+void freeStatusOrInt(StatusOrInt *status) {
+    delete status;
+}
+
 //void freeStatusOrSpeechToText(StatusOrSpeechToText *status) {
 //    delete status;
 //}
@@ -435,6 +439,16 @@ Status* graphRunnerStartCamera(CGraphRunner instance, int camera_index, ImageInf
     }
 }
 
+StatusOrInt* graphRunnerGetTimestamp(CGraphRunner instance) {
+    try {
+        auto graph_runner = reinterpret_cast<GraphRunner*>(instance);
+        return new StatusOrInt{OkStatus, "", (int)graph_runner->timestamp};
+    } catch (...) {
+        auto except = handle_exceptions();
+        return new StatusOrInt{except->status, except->message};
+    }
+}
+
 Status* graphRunnerStopCamera(CGraphRunner instance) {
     try {
         reinterpret_cast<GraphRunner*>(instance)->stop_camera();

diff --git a/openvino_bindings/src/bindings.h b/openvino_bindings/src/bindings.h
@@ -156,6 +156,7 @@ typedef void (*VLMInferenceCallbackFunction)(StatusOrString*);
 
 EXPORT void freeStatus(Status *status);
 EXPORT void freeStatusOrString(StatusOrString *status);
+EXPORT void freeStatusOrInt(StatusOrInt *status);
 EXPORT void freeStatusOrImageInference(StatusOrImageInference *status);
 EXPORT void freeStatusOrLLMInference(StatusOrLLMInference *status);
 EXPORT void freeStatusOrSpeechToText(StatusOrSpeechToText *status);
@@ -203,6 +204,7 @@ EXPORT Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const
 EXPORT StatusOrString* graphRunnerGet(CGraphRunner instance);
 EXPORT Status* graphRunnerStop(CGraphRunner instance);
 EXPORT Status* graphRunnerStartCamera(CGraphRunner instance, int cameraIndex, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay);
+EXPORT StatusOrInt* graphRunnerGetTimestamp(CGraphRunner instance);
 EXPORT Status* graphRunnerStopCamera(CGraphRunner instance);
 
 EXPORT StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device);

diff --git a/openvino_bindings/src/mediapipe/graph_runner.cc b/openvino_bindings/src/mediapipe/graph_runner.cc
@@ -40,10 +40,10 @@ void GraphRunner::stop() {
 void GraphRunner::open_camera(int deviceIndex, SerializationOutput serializationOutput, const std::function<void(std::string output)>& callback) {
     camera_handler = std::make_shared<CameraHandler>(deviceIndex);
     auto lambda_callback = [this, callback, serializationOutput](cv::Mat frame) {
+        timestamp++;
         queue("input", timestamp, frame);
         queue("serialization_output", timestamp, serializationOutput);
         callback(get());
-        timestamp++;
     };
 
     camera_handler->open_camera(lambda_callback);

diff --git a/openvino_bindings/src/mediapipe/graph_runner.h b/openvino_bindings/src/mediapipe/graph_runner.h
@@ -29,6 +29,8 @@ class GraphRunner  {
     if (timestamp > this->timestamp) {
       this->timestamp = timestamp;
     }
+
+    std::cout << "timestamp: " << timestamp << std::endl;
     auto packet = mediapipe::MakePacket<T>(content).At(mediapipe::Timestamp(timestamp));
     graph->AddPacketToInputStream(name, packet);
   }

diff --git a/openvino_bindings/src/utils/camera_handler.cc b/openvino_bindings/src/utils/camera_handler.cc
@@ -1,3 +1,8 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include "camera_handler.h"
 
 

diff --git a/openvino_bindings/src/utils/camera_handler.h b/openvino_bindings/src/utils/camera_handler.h
@@ -1,3 +1,8 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef CAMERA_HANDLER_H_
 #define CAMERA_HANDLER_H_