Merge pull request #1203 from luxonis/v3_benchmark_nodes

Benchmark nodes

CMakeLists.txt

@@ -368,7 +368,8 @@ set(TARGET_CORE_SOURCES
     src/pipeline/node/DetectionNetwork.cpp
     src/pipeline/node/Script.cpp
     src/pipeline/node/Pool.cpp
-    src/pipeline/node/Benchmark.cpp
+    src/pipeline/node/BenchmarkIn.cpp
+    src/pipeline/node/BenchmarkOut.cpp
     src/pipeline/node/SpatialDetectionNetwork.cpp
     src/pipeline/node/SystemLogger.cpp
     src/pipeline/node/SpatialLocationCalculator.cpp

bindings/python/src/pipeline/node/BenchmarkBindings.cpp

@@ -30,9 +30,13 @@ void bind_benchmark(pybind11::module& m, void* pCallstack) {
     benchmarkOut.def_readonly("out", &BenchmarkOut::out, DOC(dai, node, BenchmarkOut, out))
         .def_readonly("input", &BenchmarkOut::input, DOC(dai, node, BenchmarkOut, input))
         .def("setNumMessagesToSend", &BenchmarkOut::setNumMessagesToSend, py::arg("num"), DOC(dai, node, BenchmarkOut, setNumMessagesToSend))
+        .def("setRunOnHost", &BenchmarkOut::setRunOnHost, py::arg("runOnHost"), DOC(dai, node, BenchmarkOut, setRunOnHost))
         .def("setFps", &BenchmarkOut::setFps, py::arg("fps"), DOC(dai, node, BenchmarkOut, setFps));
     benchmarkIn.def_readonly("input", &BenchmarkIn::input, DOC(dai, node, BenchmarkIn, input))
         .def_readonly("report", &BenchmarkIn::report, DOC(dai, node, BenchmarkIn, report))
         .def_readonly("passthrough", &BenchmarkIn::passthrough, DOC(dai, node, BenchmarkIn, passthrough))
-        .def("setNumMessagesToGet", &BenchmarkIn::setNumMessagesToGet, py::arg("num"), DOC(dai, node, BenchmarkIn, setNumMessagesToGet));
+        .def("setRunOnHost", &BenchmarkIn::setRunOnHost, py::arg("runOnHost"), DOC(dai, node, BenchmarkIn, setRunOnHost))
+        .def("logReportsAsWarnings", &BenchmarkIn::logReportsAsWarnings, py::arg("logReportsAsWarnings"), DOC(dai, node, BenchmarkIn, logReportsAsWarnings))
+        .def("measureIndividualLatencies", &BenchmarkIn::measureIndividualLatencies, py::arg("attachLatencies"), DOC(dai, node, BenchmarkIn, measureIndividualLatencies))
+        .def("sendReportEveryNMessages", &BenchmarkIn::sendReportEveryNMessages, py::arg("num"), DOC(dai, node, BenchmarkIn, sendReportEveryNMessages));
 }

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
 # set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+81617bcfe7b7da9eda9654b5b3d3d3254b59a47d")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+19b67f81b54c146d079d2cbd4485fa153337dc5a")

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "c3e98b39b6a5445b2187b4109d03a146c6df37dd")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "5e016a328ac84324fb3c6bd8904141191f29dc2e")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/python/Benchmark/benchmark_camera.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+import depthai as dai
+import time
+
+# Create pipeline
+with dai.Pipeline() as pipeline:
+    # Create the nodes
+    cam = pipeline.create(dai.node.Camera).build()
+    benchmarkIn = pipeline.create(dai.node.BenchmarkIn)
+    # benchmarkIn.setRunOnHost(True) # The node can also run on host and include the transfer limitation, default is False
+    output = cam.requestFullResolutionOutput()
+    output.link(benchmarkIn.input)
+
+    pipeline.start()
+    while pipeline.isRunning():
+        time.sleep(1) # Let the logger print out the FPS

examples/python/Benchmark/benchmark_nn.py

@@ -0,0 +1,51 @@
+import depthai as dai
+import numpy as np
+
+
+# First prepare the model for benchmarking
+device = dai.Device()
+modelPath = dai.getModelFromZoo(dai.NNModelDescription("yolov6-nano", platform=device.getPlatformAsString()))
+modelArhive = dai.NNArchive(modelPath)
+inputSize = modelArhive.getInputSize()
+type = modelArhive.getConfig().model.inputs[0].preprocessing.daiType
+
+if type:
+    try:
+        frameType = dai.ImgFrame.Type.__getattribute__(type)
+    except AttributeError:
+        type = None
+
+if not type:
+    if device.getPlatform() == dai.Platform.RVC2:
+        frameType = dai.ImgFrame.Type.BGR888p
+    else:
+        frameType = dai.ImgFrame.Type.BGR888i
+
+
+# Construct the input (white) image for benchmarking
+img = np.ones((inputSize[1], inputSize[0], 3), np.uint8) * 255
+inputFrame = dai.ImgFrame()
+inputFrame.setCvFrame(img, frameType)
+
+with dai.Pipeline(device) as p:
+    benchmarkOut = p.create(dai.node.BenchmarkOut)
+    benchmarkOut.setRunOnHost(False) # The node can run on host or on device
+    benchmarkOut.setFps(-1) # As fast as possible
+
+    neuralNetwork = p.create(dai.node.NeuralNetwork).build(benchmarkOut.out, modelArhive)
+
+    benchmarkIn = p.create(dai.node.BenchmarkIn)
+    benchmarkIn.setRunOnHost(False) # The node can run on host or on device
+    benchmarkIn.sendReportEveryNMessages(100)
+    benchmarkIn.logReportsAsWarnings(False)
+    neuralNetwork.out.link(benchmarkIn.input)
+
+    outputQueue = benchmarkIn.report.createOutputQueue()
+    inputQueue = benchmarkOut.input.createInputQueue()
+
+    p.start()
+    inputQueue.send(inputFrame) # Send the input image only once
+    while p.isRunning():
+        benchmarkReport = outputQueue.get()
+        assert isinstance(benchmarkReport, dai.BenchmarkReport)
+        print(f"FPS is {benchmarkReport.fps}")

examples/python/Benchmark/benchmark_simple.py

@@ -0,0 +1,29 @@
+import depthai as dai
+
+with dai.Pipeline(createImplicitDevice=False) as p:
+    # Create a BenchmarkOut node
+    # It will listen on the input to get the first message and then send it out at a specified rate
+    # The node sends the same message out (creates new pointers), not deep copies.
+    benchmarkOut = p.create(dai.node.BenchmarkOut)
+    benchmarkOut.setRunOnHost(True) # The node can run on host or on device
+    benchmarkOut.setFps(30)
+
+    # Create a BenchmarkIn node
+    # This node is receiving the messages on the input and measuring the FPS and latency.
+    # In the case that the input is with BenchmarkOut, the latency measurement is not always possible, as the message is not deep copied,
+    # which means that the timestamps stay the same and latency virtually increases over time.
+    benchmarkIn = p.create(dai.node.BenchmarkIn)
+    benchmarkIn.setRunOnHost(True) # The node can run on host or on device
+    benchmarkIn.sendReportEveryNMessages(100)
+
+    benchmarkOut.out.link(benchmarkIn.input)
+    outputQueue = benchmarkIn.report.createOutputQueue()
+    inputQueue = benchmarkOut.input.createInputQueue()
+
+    p.start()
+    imgFrame = dai.ImgFrame()
+    inputQueue.send(imgFrame)
+    while p.isRunning():
+        benchmarkReport = outputQueue.get()
+        assert isinstance(benchmarkReport, dai.BenchmarkReport)
+        print(f"FPS is {benchmarkReport.fps}")

examples/python/CMakeLists.txt

@@ -40,6 +40,7 @@ function(add_python_example example_name python_script_path)
         # Python path (to find compiled module)
         "PYTHONPATH=$<TARGET_FILE_DIR:${TARGET_NAME}>${SYS_PATH_SEPARATOR}$ENV{PYTHONPATH}"
         "DEPTHAI_SEARCH_TIMEOUT=15000"
+        "DEPTHAI_CONNECT_TIMEOUT=15000"
         "DEPTHAI_RECONNECT_TIMEOUT=0"
         # ASAN in case of sanitizers
         "${ASAN_ENVIRONMENT_VARS}"
@@ -60,6 +61,7 @@ function(add_python_example example_name python_script_path)
             # Python path (to find compiled module)
             "PYTHONPATH=$<TARGET_FILE_DIR:${TARGET_NAME}>${SYS_PATH_SEPARATOR}$ENV{PYTHONPATH}"
             "DEPTHAI_SEARCH_TIMEOUT=30000"
+            "DEPTHAI_CONNECT_TIMEOUT=30000"
             "DEPTHAI_RECONNECT_TIMEOUT=0"
             # ASAN in case of sanitizers
             ${ASAN_ENVIRONMENT_VARS}
@@ -223,3 +225,15 @@ set_tests_properties(py_script_simple PROPERTIES FAIL_REGULAR_EXPRESSION "\\[err
 
 add_python_example(script_all_cameras Script/script_switch_all_cameras.py)
 dai_set_example_test_labels(script_all_cameras ondevice rvc2_all rvc4 ci)
+
+## Benchmark node
+add_python_example(benchmark_node Benchmark/benchmark_simple.py)
+dai_set_example_test_labels(benchmark_node ondevice rvc2_all rvc4 ci)
+set_tests_properties(py_benchmark_node PROPERTIES FAIL_REGULAR_EXPRESSION "\\[error\\];\\[critical\\]")
+
+add_python_example(benchmark_cameras Benchmark/benchmark_camera.py)
+dai_set_example_test_labels(benchmark_cameras ondevice rvc2_all rvc4 ci)
+set_tests_properties(py_benchmark_cameras PROPERTIES FAIL_REGULAR_EXPRESSION "\\[error\\];\\[critical\\]")
+
+add_python_example(benchmark_nn Benchmark/benchmark_nn.py)
+dai_set_example_test_labels(benchmark_nn ondevice rvc2_all rvc4 ci)

include/depthai/pipeline/datatype/BenchmarkReport.hpp

@@ -3,22 +3,20 @@
 #include "depthai/pipeline/datatype/Buffer.hpp"
 namespace dai {
 
-// TODO(before mainline) - API not supported on RVC2
 /**
  * BenchmarkReport message.
  */
 class BenchmarkReport : public Buffer {
    public:
     BenchmarkReport() = default;
-    virtual ~BenchmarkReport() = default;
 
-    float fps;
-    float timeTotal;  // seconds
-    float numMessagesReceived;
-    float averageLatency;
+    float fps = 0.0f;
+    float timeTotal = 0.0f;  // seconds
+    float numMessagesReceived = 0;
+    float averageLatency = 0.0f;  // seconds
+
+    // Only filled if measureIndividualLatencies is set to true
     std::vector<float> latencies;
-    // TODO Add jitter, timestamps for start/end, possibly a vector of timestamps for all messages
-    // TODO BEFORE MAINLINE add setters and getters
 
     void serialize(std::vector<std::uint8_t>& metadata, DatatypeEnum& datatype) const override {
         metadata = utility::serialize(*this);

include/depthai/pipeline/node/BenchmarkIn.hpp

@@ -3,13 +3,12 @@
 #include <depthai/pipeline/DeviceNode.hpp>
 
 // shared
-#include <depthai/properties/BenchmarkPropertiesIn.hpp>
+#include <depthai/properties/BenchmarkInProperties.hpp>
 
 namespace dai {
 namespace node {
 
-// TODO(before mainline) - API not supported on RVC2
-class BenchmarkIn : public DeviceNodeCRTP<DeviceNode, BenchmarkIn, BenchmarkPropertiesIn> {
+class BenchmarkIn : public DeviceNodeCRTP<DeviceNode, BenchmarkIn, BenchmarkInProperties>, public HostRunnable {
    public:
     constexpr static const char* NAME = "BenchmarkIn";
     using DeviceNodeCRTP::DeviceNodeCRTP;
@@ -30,11 +29,35 @@ class BenchmarkIn : public DeviceNodeCRTP<DeviceNode, BenchmarkIn, BenchmarkProp
     Output report{*this, {"report", DEFAULT_GROUP, {{{DatatypeEnum::BenchmarkReport, false}}}}};
 
     /**
-     * Set number of messages that the nodes retrieves before sending the report
-     * The passthrough keeps getting forwarded after the report is sent
-     * @param num of messages to get for report
+     * Specify how many messages to measure for each report
      */
-    void setNumMessagesToGet(int num);
+    void sendReportEveryNMessages(uint32_t n);
+
+    /**
+     * Specify whether to run on host or device
+     * By default, the node will run on device.
+     */
+    void setRunOnHost(bool runOnHost);
+
+    /**
+     * Check if the node is set to run on host
+     */
+    bool runOnHost() const override;
+
+    /**
+    * Log the reports as warnings
+    */
+    void logReportsAsWarnings(bool logReportsAsWarnings);
+
+    /**
+     * Attach latencies to the report
+     */
+    void measureIndividualLatencies(bool attachLatencies);
+
+    void run() override;
+
+   private:
+    bool runOnHostVar = false;
 };
 
 }  // namespace node

include/depthai/pipeline/node/BenchmarkOut.hpp

@@ -3,12 +3,12 @@
 #include <depthai/pipeline/DeviceNode.hpp>
 
 // shared
-#include <depthai/properties/BenchmarkPropertiesOut.hpp>
+#include <depthai/properties/BenchmarkOutProperties.hpp>
 
 namespace dai {
 namespace node {
 
-class BenchmarkOut : public DeviceNodeCRTP<DeviceNode, BenchmarkOut, BenchmarkPropertiesOut> {
+class BenchmarkOut : public DeviceNodeCRTP<DeviceNode, BenchmarkOut, BenchmarkOutProperties>, public HostRunnable{
    public:
     constexpr static const char* NAME = "BenchmarkOut";
     using DeviceNodeCRTP::DeviceNodeCRTP;
@@ -34,7 +34,21 @@ class BenchmarkOut : public DeviceNodeCRTP<DeviceNode, BenchmarkOut, BenchmarkPr
      */
     void setFps(float fps);
 
-    void buildInternal() override;
+    /**
+     * Specify whether to run on host or device
+     * By default, the node will run on device.
+     */
+    void setRunOnHost(bool runOnHost);
+
+    /**
+     * Check if the node is set to run on host
+     */
+    bool runOnHost() const override;
+
+    void run() override;
+
+   private:
+    bool runOnHostVar = false;
 };
 
 }  // namespace node

include/depthai/properties/BenchmarkInProperties.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "depthai/common/ProcessorType.hpp"
+#include "depthai/common/optional.hpp"
+#include "depthai/pipeline/datatype/DatatypeEnum.hpp"
+#include "depthai/properties/Properties.hpp"
+
+namespace dai {
+
+/**
+ * Specify benchmark properties (number of messages to send/receive)
+ */
+struct BenchmarkInProperties : PropertiesSerializable<Properties, BenchmarkInProperties> {
+    /**
+     * Specify how many messages to measure for each report
+     */
+    uint32_t reportEveryNMessages = 50;
+
+    /**
+    * Specify whether the latenices are attached to the report individually
+    */
+    bool attachLatencies = false;
+
+    /**
+     * Send the reports also as logger warnings
+     */
+    bool logReportsAsWarnings = true;
+};
+
+DEPTHAI_SERIALIZE_EXT(BenchmarkInProperties, reportEveryNMessages, attachLatencies, logReportsAsWarnings);
+
+}  // namespace dai

include/depthai/properties/BenchmarkPropertiesOut.hpp → include/depthai/properties/BenchmarkOutProperties.hpp

@@ -10,18 +10,18 @@ namespace dai {
 /**
  * Specify benchmark properties (number of messages to send/receive)
  */
-struct BenchmarkPropertiesOut : PropertiesSerializable<Properties, BenchmarkPropertiesOut> {
+struct BenchmarkOutProperties : PropertiesSerializable<Properties, BenchmarkOutProperties> {
     /**
      * Number of messages to send
      */
-    int numMessages = 50;
+    int numMessages = -1;
 
     /**
      * FPS for sending, 0 means as fast as possible
      */
     float fps = 0;
 };
 
-DEPTHAI_SERIALIZE_EXT(BenchmarkPropertiesOut, numMessages, fps);
+DEPTHAI_SERIALIZE_EXT(BenchmarkOutProperties, numMessages, fps);
 
 }  // namespace dai