From 0c39e43fb9ff64ef7e13ca0e25d25a488c883f7d Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 07:35:22 +0100 Subject: [PATCH 01/10] Add a dataset for PandaDemo that does not exhibit the limitation in MuJoCo simulation --- src/PandaDemoParameters.csv | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/PandaDemoParameters.csv b/src/PandaDemoParameters.csv index da58e57..a9c0bfb 100644 --- a/src/PandaDemoParameters.csv +++ b/src/PandaDemoParameters.csv @@ -24,4 +24,13 @@ dataset 2,, 0.2,3.14159,-3.14159 0.2,3.5,-1 0.2,1,-1 -20,255,0 \ No newline at end of file +20,255,0 +dataset 3,, +0.05,2.5,-2 +0.05,1.2,-0.8 +0.05,1,-1 +0.05,0,-2 +0.05,2,-2 +0.05,2.5,-0.3 +0.05,1,-1 +10,200,0 \ No newline at end of file From d13604a4faf9a412cad3311c67d931d976112b13 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 07:38:12 +0100 Subject: [PATCH 02/10] Ignore the virtualenvironment named .venv --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 89f3a61..0f0f78a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ src-gen /include fed-gen MUJOCO_LOG.TXT +.venv \ No newline at end of file From e64987f8f935dafa6ee0c5a30532387a4c30edab Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 07:39:09 +0100 Subject: [PATCH 03/10] Copy and add Yolo and video examples from the playground --- src/lib/video/README.md | 48 ++++++++++ src/lib/video/Video.lf | 96 +++++++++++++++++++ src/lib/video/VideoAsync.lf | 84 ++++++++++++++++ src/lib/video/YOLOv5_Webcam.lf | 169 +++++++++++++++++++++++++++++++++ src/lib/video/requirements.txt | 33 +++++++ 5 files changed, 430 insertions(+) create mode 100644 src/lib/video/README.md create mode 100644 src/lib/video/Video.lf create mode 100644 src/lib/video/VideoAsync.lf create mode 100644 src/lib/video/YOLOv5_Webcam.lf create mode 100644 src/lib/video/requirements.txt diff --git a/src/lib/video/README.md b/src/lib/video/README.md new file mode 100644 index 0000000..e5a2d79 --- /dev/null +++ b/src/lib/video/README.md @@ -0,0 +1,48 @@ +# Video and YOLO Library + +This library provides video capture, display, and DNN-based object recognition using YOLOv5 for use in the `PandaDemoCamCtrl.lf` programs. + +## Setup + +### 1. Install PyTorch + +Go to the [PyTorch website](https://pytorch.org/get-started/locally/) and follow the instructions for your platform. + +**If you have an NVIDIA GPU**, select the correct CUDA version on the installation page and follow the generated command, for example: + +```bash +pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121 +``` + +**If you have no GPU (CPU only)**, install the CPU builds explicitly. Using the generic PyPI packages will cause a version mismatch and a segfault at runtime: + +```bash +pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu +``` + +### 2. Install remaining dependencies + +```bash +pip install -r requirements.txt +``` + +> **Note:** `requirements.txt` intentionally omits `torch` and `torchvision` so that the correct build variant (CPU or CUDA) can be chosen in the step above. + +### 3. Font warning fix (OpenCV / Qt) + +If you see repeated `QFontDatabase: Cannot find font directory` warnings, Qt's bundled font directory is missing. Fix it by symlinking system DejaVu fonts into the expected location: + +```bash +mkdir -p "$(python3 -c 'import cv2, os; print(os.path.dirname(cv2.__file__))')/qt/fonts" +ln -s /usr/share/fonts/truetype/dejavu/*.ttf \ + "$(python3 -c 'import cv2, os; print(os.path.dirname(cv2.__file__))')/qt/fonts/" +``` + +### 4. Compile and run + +Compile with `lfc` and run the generated Python program: + +```bash +lfc src/PandaDemoCamCtrl.lf +bin/PandaDemoCamCtrl +``` diff --git a/src/lib/video/Video.lf b/src/lib/video/Video.lf new file mode 100644 index 0000000..78a95f8 --- /dev/null +++ b/src/lib/video/Video.lf @@ -0,0 +1,96 @@ +/** + * Copied from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf + * Original license: [BSD 2-Clause License] + * See README.md for instructions on how to install the required libraries. + * Date copied: 2026-05-01 + * + * Two modifications from the original: + * 1. Added support for IP camera streams via the url parameter of WebCam. + * 2. Added Minimal buffering to reduce latency. +**/ + +/** Video capture and playback example using OpenCV. Please see README.md for instructions. */ +target Python { + single-threaded: true # OpenCV crashes if we use the multithreaded version. +} + +preamble {= + import cv2 +=} + +/** + * Produce a sequence of frames with the specified offset and period. + * @param webcam_id The ID of the camera (default 0). + * @param url An optional URL to an IP camera stream. If specified, this overrides webcam_id. + * @param offset Time until frames start to be captured. + * @param period The period with which frames will be read. + */ +reactor WebCam(webcam_id=0, url="", offset = 0 s, period = 100 ms) { + output camera_frame + + state stream + timer camera_tick(offset, period) + + reaction(startup) {= + if (self.url != ""): + self.stream = cv2.VideoCapture(self.url) + else: + self.stream = cv2.VideoCapture(self.webcam_id, cv2.CAP_ANY) # or CAP_DSHOW + + # Minimal buffering to reduce latency. + self.stream.set(cv2.CAP_PROP_BUFFERSIZE, 1) + + if not self.stream.isOpened(): + sys.stderr.write("Error: Failed to capture from the webcam.\n") + exit(1) + + # Here, LF is in charge of the timing, so do not set the frame rate. + # self.stream.set(cv2.CAP_PROP_FPS, 30) # Set the camera's FPS to 30 + =} + + reaction(camera_tick) -> camera_frame {=# Minimal buffering to reduce latency. + # read() is a combination of grab() and retrieve(). + ret, frame = self.stream.read() + if ret: + camera_frame.set(frame) + else: + print("WARNING: Camera frame missing.") + =} + + reaction(shutdown) {= + self.stream.release() + =} +} + +/** Display video frames. */ +reactor Display { + input frame + state frame_count = 0 + + reaction(startup) {= + print("\n******* Press 'q' in the video window to exit *******\n") + =} + + reaction(frame) {= + self.frame_count += 1 + # Every hundred or so frames, report the frame rate. + if (self.frame_count % 100 == 0): + print(f"** Average frame rate: {self.frame_count * SEC(1) / lf.time.physical_elapsed()} f/s") + + cv2.imshow("frame", frame.value) + # press 'Q' if you want to exit + if cv2.waitKey(1) & 0xFF == ord('q'): + request_stop() + =} + + reaction(shutdown) {= + # Destroy the all windows now + cv2.destroyAllWindows() + =} +} + +main reactor { + webcam = new WebCam(webcam_id = 1) + display = new Display() + webcam.camera_frame -> display.frame +} diff --git a/src/lib/video/VideoAsync.lf b/src/lib/video/VideoAsync.lf new file mode 100644 index 0000000..45696a9 --- /dev/null +++ b/src/lib/video/VideoAsync.lf @@ -0,0 +1,84 @@ +/** + * Copied from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/VideoAsync.lf + * Original license: [BSD 2-Clause License] + * See README.md for instructions on how to install the required libraries. + * Date copied: 2026-05-01 +**/ + +/** + * Video capture and playback example using OpenCV with the camera driving the timing. Please see + * README.md for instructions. + */ +target Python { + keepalive: true, + single-threaded: true # OpenCV crashes if we use the multithreaded version. +} + +import Display from "Video.lf" + +preamble {= + import cv2 + import time +=} + +/** + * Produce a sequence of frames as they are delivered by the camera. This version uses blocking + * reads to read a video frame and starts the read shortly after completing the previous read. This + * version should only be used in programs where the camera frames drive everything because the + * WebCamAsync will block until it gets a camera frame. + * + * @param webcam_id The ID of the camera (default 0). + * @param offset Time until frames start to be captured. + * @param frames_per_second The number of frames per second to set the camera to. + */ +reactor WebCamAsync(webcam_id=0, offset = 0 s, frames_per_second=30) { + input trigger + output camera_frame + + timer start(offset) + state stream + + reaction(start) -> camera_frame {= + self.stream = cv2.VideoCapture(self.webcam_id, cv2.CAP_ANY) + if (self.stream.isOpened() is not True): + sys.stderr.write("Error: Failed to open the camera.\n") + exit(1) + + self.stream.set(cv2.CAP_PROP_FPS, self.frames_per_second) + + # macOS AVFoundation needs a moment to warm up after open() before + # frames are available; retry for up to ~2 seconds before giving up. + ret, frame = False, None + for _ in range(20): + ret, frame = self.stream.read() + if ret: + break + time.sleep(0.1) + + if ret: + camera_frame.set(frame) + else: + sys.stderr.write("Error: Failed to get first frame from camera.\n") + exit(1) + =} + + reaction(trigger) -> camera_frame {= + # Read a frame. This is a blocking read. + ret, frame = self.stream.read() + if ret is True: + camera_frame.set(frame) + else: + print("Warning, failed to get frame.") + =} + + reaction(shutdown) {= + self.stream.release() + =} +} + +main reactor { + webcam = new WebCamAsync() + display = new Display() + webcam.camera_frame -> display.frame + webcam.camera_frame ~> webcam.trigger +} diff --git a/src/lib/video/YOLOv5_Webcam.lf b/src/lib/video/YOLOv5_Webcam.lf new file mode 100644 index 0000000..691baf2 --- /dev/null +++ b/src/lib/video/YOLOv5_Webcam.lf @@ -0,0 +1,169 @@ +/** + * Copied from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf + * Original license: [BSD 2-Clause License] + * See README.md for instructions on how to install the required libraries. + * Date copied: 2026-05-01 + * + * One modification from the original: + * 1. Removed the deadline on the DNN, for latency purposes. +**/ + + +/** + * @brief Example of a Deep Neural Network (YOLOv5) in LF. + * + * Please see README.md for instructions. This uses ultralytics/yolov5. Adapted from: + * https://towardsdatascience.com/implementing-real-time-object-detection-system-using-pytorch-and-opencv-70bac41148f7 + */ +target Python { + keepalive: true, + single-threaded: true # OpenCV crashes if we use the multithreaded version. +} + +import WebCamAsync from "VideoAsync.lf" +import Display from "Video.lf" + +preamble {= + BILLION = 1_000_000_000 + import cv2 +=} + +/** + * A YOLOv5 DNN that takes a frame as input and produces object 'labels' and object label + * coordinates (where each label/object is on the frame). + */ +reactor DNN { + input frame # Image input frame + + output labels # Label outputs + output label_coordinates # Label coordinates + output model # Send the model to anyone who's interested + + state _model # The DNN model + state _device # The device to use (e.g., cpu or cuda) + preamble {= + import torch + from torch import hub + =} + + reaction(startup) -> model {= + # YOLOv5 hub code calls the deprecated torch.cuda.amp.autocast on every + # inference. Redirect it to the current API so the FutureWarning disappears. + self.torch.cuda.amp.autocast = lambda enabled=True, **kw: self.torch.amp.autocast("cuda", enabled=enabled, **kw) + + # PyTorch 2.6 changed torch.load to default weights_only=True, which breaks + # the YOLOv5 checkpoint format. Patch it back to False for this trusted load. + _orig_load = self.torch.load + self.torch.load = lambda *a, **kw: _orig_load(*a, **{**kw, "weights_only": kw.get("weights_only", False)}) + try: + self._model = self.torch.hub.load("ultralytics/yolov5", "yolov5s", pretrained=True) + finally: + self.torch.load = _orig_load + # Find out if CUDA is supported + self._device = "cuda" if self.torch.cuda.is_available() else 'cpu' + # Send the model to device + self._model.to(self._device) + # Send the model to whoever is interested (other reactors) + model.set(self._model) + =} + + reaction(frame) -> labels, label_coordinates {= + # Convert the frame into a tuple + fr = [frame.value] + # Run the model on the frame + results = self._model(fr) + # Extract the labels + labels.set(results.xyxyn[0][:, -1].cpu().numpy()) + # Extract the coordinates for the label + label_coordinates.set(results.xyxyn[0][:, :-1].cpu().numpy()) + =} +} + +/** Plot frames with labels superimposed on top of each object in the frame. */ +reactor Plotter(label_deadline = 100 msec) { + input frame + input labels + input label_coordinates + input model + + output result + + state _model # Keep the model + state _prev_time = 0 + + /** Receive the DNN model */ + reaction(model) {= + self._model = model.value + =} + + // /** Impose a deadline on object labels */ + // reaction(labels) {= + // # DNN output was on time + // =} deadline(label_deadline) {= + // print(f"Received the DNN output late by about {(lf.time.physical() - lf.time.logical())/1000000}ms.") + // =} + + /** + * Given a frame, object labels, and the corresponding object label coordinates, draw on the frame + * and produce an output. + */ + reaction(frame, labels, label_coordinates) -> result {= + if (not frame.is_present or + not labels.is_present or + not label_coordinates.is_present): + sys.stderr.write("Error: Expected all inputs to be present at the same time.\n") + request_stop() + return + + # Get how many labels we have + n = len(labels.value) + x_shape, y_shape = frame.value.shape[1], frame.value.shape[0] + for i in range(n): + row = label_coordinates.value[i] + # If score is less than 0.2 we avoid making a prediction. + if row[4] < 0.2: + continue + x1 = int(row[0]*x_shape) + y1 = int(row[1]*y_shape) + x2 = int(row[2]*x_shape) + y2 = int(row[3]*y_shape) + bgr = (0, 255, 0) # color of the box + classes = self._model.names # Get the name of label index + label_font = cv2.FONT_HERSHEY_SIMPLEX #Font for the label. + cv2.rectangle(frame.value, \ + (x1, y1), (x2, y2), \ + bgr, 2) #Plot the boxes + cv2.putText(frame.value,\ + classes[int(labels.value[i])], \ + (x1, y1), \ + label_font, 0.9, bgr, 2) #Put a label over box. + + result.set(frame.value) + =} + + reaction(shutdown) {= + # Destroy the all windows now + cv2.destroyAllWindows() + =} +} + +main reactor { + # Offset allows time for the model to load. + webcam = new WebCamAsync(webcam_id=0, offset = 2 s) + dnn = new DNN() + plotter = new Plotter() + display = new Display() + + # Send the camera frame to the DNN to be process and to the plotter to be depicted + (webcam.camera_frame)+ -> dnn.frame, plotter.frame + # Send outputs of the DNN (object labels and their coordinates) to the plotter + dnn.labels, dnn.label_coordinates -> plotter.labels, plotter.label_coordinates + + # Send the DNN model to the plotter. It will be used to extract the human-readable names + # of each label. + dnn.model -> plotter.model + + webcam.camera_frame ~> webcam.trigger + + plotter.result -> display.frame +} diff --git a/src/lib/video/requirements.txt b/src/lib/video/requirements.txt new file mode 100644 index 0000000..10352b2 --- /dev/null +++ b/src/lib/video/requirements.txt @@ -0,0 +1,33 @@ +# MuJoCo +mujoco + +# NOTE: torch and torchvision are NOT listed here. +# Install them separately BEFORE running pip install -r requirements.txt. +# See README.md for the correct command depending on whether you have a GPU. +# +# CPU (no GPU): +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu +# +# NVIDIA GPU (example for CUDA 12.1): +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121 + + +# Video processing and analysis +opencv-python>=4.8.0 +pandas +tqdm +seaborn +ipywidgets +ipython +ipykernel +matplotlib>=3.2.2 +numpy>=2.0.0 +Pillow>=7.1.2 +PyYAML>=5.3.1 +scipy>=1.4.1 +requests +ultralytics + +# Logging +tensorboard>=2.4.1 +thop From b728e9a6ac4382e1fee1bad4cc164c640e2b2a68 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:03:06 +0100 Subject: [PATCH 04/10] Ignore yolo model files --- .gitignore | 3 ++- src/lib/video/{YOLOv5_Webcam.lf => YOLOv8_Webcam.lf} | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename src/lib/video/{YOLOv5_Webcam.lf => YOLOv8_Webcam.lf} (100%) diff --git a/.gitignore b/.gitignore index 0f0f78a..543320d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ src-gen /include fed-gen MUJOCO_LOG.TXT -.venv \ No newline at end of file +.venv +*.pt \ No newline at end of file diff --git a/src/lib/video/YOLOv5_Webcam.lf b/src/lib/video/YOLOv8_Webcam.lf similarity index 100% rename from src/lib/video/YOLOv5_Webcam.lf rename to src/lib/video/YOLOv8_Webcam.lf From 90299a3b69434c44828f08c98c06ddf899766e2d Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:12:52 +0100 Subject: [PATCH 05/10] Add a comment on the use of python or mjpython --- src/lib/MuJoCoBase.lf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/MuJoCoBase.lf b/src/lib/MuJoCoBase.lf index 4c68731..b376a63 100644 --- a/src/lib/MuJoCoBase.lf +++ b/src/lib/MuJoCoBase.lf @@ -1,9 +1,11 @@ /** @file Base class for reactors using the MuJoCo physics-based simulation framework. */ target Python { keepalive: true, - cmake-args: { - Python_EXECUTABLE: "mjpython" - } + single-threaded: true, + // If you are using `mjpython`, then uncomment the following lines + // cmake-args: { + // Python_EXECUTABLE: "mjpython" + // } } preamble {= From 85413a35bec16d75c64582d9d290e58babb71f67 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:13:37 +0100 Subject: [PATCH 06/10] More requirements for video processing --- src/lib/video/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/video/requirements.txt b/src/lib/video/requirements.txt index 10352b2..262cdff 100644 --- a/src/lib/video/requirements.txt +++ b/src/lib/video/requirements.txt @@ -31,3 +31,8 @@ ultralytics # Logging tensorboard>=2.4.1 thop + +# rendering +PyOpenGL +PyOpenGL-accelerate +PyQt5 \ No newline at end of file From 5351cbd3fb64bdc09e490e7677ee7027a7546d85 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:14:58 +0100 Subject: [PATCH 07/10] Move to YOLOv8 instead of YOLOv5, because it is way faster --- src/lib/video/YOLOv8_Webcam.lf | 59 +++++++++++++--------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/src/lib/video/YOLOv8_Webcam.lf b/src/lib/video/YOLOv8_Webcam.lf index 691baf2..ef13d59 100644 --- a/src/lib/video/YOLOv8_Webcam.lf +++ b/src/lib/video/YOLOv8_Webcam.lf @@ -1,20 +1,17 @@ /** - * Copied from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf + * This a variant of from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf, but using YOLOv8 instead of YOLOv5. The code is adapted from: + * https://towardsdatascience.com/implementing-real-time-object-detection-system-using-pytorch-and-opencv-70bac41148f7 + * * Original license: [BSD 2-Clause License] * See README.md for instructions on how to install the required libraries. * Date copied: 2026-05-01 - * + * + * Please see README.md for instructions. This uses ultralytics/yolov8. Adapted from: + * * One modification from the original: * 1. Removed the deadline on the DNN, for latency purposes. **/ - -/** - * @brief Example of a Deep Neural Network (YOLOv5) in LF. - * - * Please see README.md for instructions. This uses ultralytics/yolov5. Adapted from: - * https://towardsdatascience.com/implementing-real-time-object-detection-system-using-pytorch-and-opencv-70bac41148f7 - */ target Python { keepalive: true, single-threaded: true # OpenCV crashes if we use the multithreaded version. @@ -29,7 +26,7 @@ preamble {= =} /** - * A YOLOv5 DNN that takes a frame as input and produces object 'labels' and object label + * A YOLOv8 DNN that takes a frame as input and produces object 'labels' and object label * coordinates (where each label/object is on the frame). */ reactor DNN { @@ -43,39 +40,27 @@ reactor DNN { state _device # The device to use (e.g., cpu or cuda) preamble {= import torch - from torch import hub + import numpy as np + from ultralytics import YOLO =} reaction(startup) -> model {= - # YOLOv5 hub code calls the deprecated torch.cuda.amp.autocast on every - # inference. Redirect it to the current API so the FutureWarning disappears. - self.torch.cuda.amp.autocast = lambda enabled=True, **kw: self.torch.amp.autocast("cuda", enabled=enabled, **kw) - - # PyTorch 2.6 changed torch.load to default weights_only=True, which breaks - # the YOLOv5 checkpoint format. Patch it back to False for this trusted load. - _orig_load = self.torch.load - self.torch.load = lambda *a, **kw: _orig_load(*a, **{**kw, "weights_only": kw.get("weights_only", False)}) - try: - self._model = self.torch.hub.load("ultralytics/yolov5", "yolov5s", pretrained=True) - finally: - self.torch.load = _orig_load - # Find out if CUDA is supported - self._device = "cuda" if self.torch.cuda.is_available() else 'cpu' - # Send the model to device - self._model.to(self._device) - # Send the model to whoever is interested (other reactors) + self._model = self.YOLO("yolov8n.pt") + self._device = "cuda" if self.torch.cuda.is_available() else "cpu" model.set(self._model) =} reaction(frame) -> labels, label_coordinates {= - # Convert the frame into a tuple - fr = [frame.value] - # Run the model on the frame - results = self._model(fr) - # Extract the labels - labels.set(results.xyxyn[0][:, -1].cpu().numpy()) - # Extract the coordinates for the label - label_coordinates.set(results.xyxyn[0][:, :-1].cpu().numpy()) + results = self._model(frame.value, verbose=False) + boxes = results[0].boxes + if len(boxes) > 0: + xyxyn = boxes.xyxyn.cpu().numpy() + conf = boxes.conf.cpu().numpy().reshape(-1, 1) + labels.set(boxes.cls.cpu().numpy()) + label_coordinates.set(self.np.hstack([xyxyn, conf])) + else: + labels.set(self.np.empty(0)) + label_coordinates.set(self.np.empty((0, 5))) =} } @@ -96,7 +81,7 @@ reactor Plotter(label_deadline = 100 msec) { self._model = model.value =} - // /** Impose a deadline on object labels */ + /** Impose a deadline on object labels */ // reaction(labels) {= // # DNN output was on time // =} deadline(label_deadline) {= From 36bbe7708be9cfa41a55756e7ea2730d84e04eb0 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:15:29 +0100 Subject: [PATCH 08/10] Add cam controlled panda demo --- src/PandaDemoCamCtrl.lf | 162 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 src/PandaDemoCamCtrl.lf diff --git a/src/PandaDemoCamCtrl.lf b/src/PandaDemoCamCtrl.lf new file mode 100644 index 0000000..174339d --- /dev/null +++ b/src/PandaDemoCamCtrl.lf @@ -0,0 +1,162 @@ +/** + * Franka Emika Panda robot demo with camera-based person detection. + * + * Moves each of the eight joints of the Panda robot through a range of values defined in a CSV + * file, while using a YOLOv8 DNN to monitor the webcam feed. The robot pauses automatically + * when two or more people are detected nearby, and resumes when the area is clear. + * + * See [README.md](../README.md) for prerequisites and installation instructions. + * + * The CSV file "PandaDemoParameters.csv" (located in the same directory) defines motion + * parameters. There are three datasets corresponding to three different speeds of motion. + * To select a dataset, change the top-level `dataset` parameter to 0, 1, or 2. + * + * The Robot arm will stop moving when the number of people detected by the DNN is greater than + * or equal to 2. The robot will resume moving when the number of people detected is less than 2. + * + * The demo is based on the original Panda demo, as well as Yolo, DNN and Video examples from the + * LF playground. + * + * @author Chadlia Jerad + */ + +target Python { + single-threaded: true, + keepalive: true +} + +import MuJoCoPanda from "lib/MuJoCoPanda.lf" +import DNN, Plotter from "lib/video/YOLOv8_Webcam.lf" +import WebCam, Display from "lib/video/Video.lf" + +preamble {= + import csv + from mujoco.glfw import glfw +=} + +reactor SimpleController( + period = 100 ms, + dataset = 0, + step=0.1, + high_limit=1.0, + low_limit=-1.0, + bank_index=0, + parameter_file="") +{ + timer t(0, period) + input restart + input pause_play + output command + state motor = 0.0 + + # In Python, can't overwrite parameter values, so we use state variables instead. + state step_value = step + state high_limit_value = high_limit + state low_limit_value = low_limit + + reaction(startup) {= + if self.parameter_file: + with open(self.parameter_file, "r", encoding="utf-8-sig") as f: + rows = list(csv.reader(f)) + row_idx = (9 * self.dataset) + self.bank_index + 1 + if row_idx < len(rows): + row = rows[row_idx] + self.step_value = float(row[0].strip()) + self.high_limit_value = float(row[1].strip()) + self.low_limit_value = float(row[2].strip()) + =} + + initial mode PLAY { + reaction(restart) {= + self.motor = 0.0 + =} + + reaction(pause_play) -> PAUSE {= + PAUSE.set() + =} + + reaction(t) -> command {= + command.set(self.motor) + self.motor += self.step_value + if self.motor > self.high_limit_value or self.motor < self.low_limit_value: + self.step_value = -self.step_value + =} + } + + mode PAUSE { + reaction(t) -> command {= + command.set(0.0) + =} + + reaction(pause_play) -> reset(PLAY) {= + PLAY.set() + =} + } +} + +main reactor(dataset = 3) { + timer read_state_timer(0, 500 ms) + state nbr_person_detected = 0 + state is_paused = False + + m = new MuJoCoPanda(frame_period = 50 ms) + c = new[8] SimpleController( + period = 100 ms, + dataset = dataset, + parameter_file = {= lf.source_directory() + "/PandaDemoParameters.csv" =} + ) + # Offset prevents deadline violations during startup. + webcam = new WebCam(webcam_id=0, offset = 3 s) + // webcam = new WebCam(url="http://10.152.67.28:8080/video", offset = 3 s) + dnn = new DNN() + plotter = new Plotter(label_deadline = 100 msec) + display = new Display() + + c.command -> m.motor + (webcam.camera_frame)+ -> dnn.frame, plotter.frame + dnn.labels, dnn.label_coordinates -> plotter.labels, plotter.label_coordinates + + dnn.model -> plotter.model + plotter.result -> display.frame + + reaction(startup) {= + print("*** Backspace to reset.") + print("*** Type q to quit.\n") + =} + + reaction(m.key) -> m.restart, c.restart {= + if m.key.value.act == glfw.PRESS: + if m.key.value.key == glfw.KEY_BACKSPACE: + m.restart.set(True) + for i in range(8): + c[i].restart.set(True) + elif m.key.value.key == glfw.KEY_Q: + lf.request_stop() + =} + + reaction(read_state_timer) -> m.read_state {= + m.read_state.set(True) + =} + + reaction(dnn.labels) -> c.pause_play{= + person_count = sum(1 for label in dnn.labels.value if int(label) == 0) + self.nbr_person_detected = person_count + should_pause = person_count >= 2 + if should_pause and not self.is_paused: + print(f"{person_count} persons detected at {lf.time.logical_elapsed()/1000000000} => Pausing robot: too many people around!") + for i in range(8): + c[i].pause_play.set(True) + self.is_paused = True + elif not should_pause and self.is_paused: + print(f"{person_count} persons detected at {lf.time.logical_elapsed()/1000000000} => Resuming robot.") + for i in range(8): + c[i].pause_play.set(True) + self.is_paused = False + =} + + reaction(m.joint_pos, m.joint_vel) {= + qpos = " ".join(f"{m.joint_pos[i].value:.4f}" for i in range(9)) + qvel = " ".join(f"{m.joint_vel[i].value:.4f}" for i in range(9)) + # print(f"qpos={qpos} | qvel={qvel}") + =} +} From f2a35e41b317db47e4f8a9496f082bcb4bf1cf43 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:21:06 +0100 Subject: [PATCH 09/10] Update Readme file to account for PandaDemoCanCtrl, and instructions on running it --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index cafe922..e10c6ce 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,28 @@ Build the demos using `lfc` or `make`: * [MuJoCoCarDemo](src/MuJoCoCarDemo.lf): Simple drivable car. * [MuJoCoCarAutoDemo](src/MuJoCoCarAutoDemo.lf): Simple drivable car. * [PandaDemo](src/PandaDemo.lf): Franka Emika Panda robot doing gyrations. +* [PandaDemoCamCtrl](src/PandaDemoCamCtrl.lf): Franka Emika Panda robot with camera-based person detection. The robot pauses automatically when two or more people are detected by a YOLOv8 DNN monitoring the webcam feed, and resumes when the area is clear. + +### PandaDemoCamCtrl Prerequisites + +This demo uses [YOLOv8](https://github.com/ultralytics/ultralytics) and OpenCV for real-time object detection. It requires a Python virtual environment with the dependencies listed in [src/lib/video/requirements.txt](src/lib/video/requirements.txt), which covers all reactors under [src/lib/video/](src/lib/video/). + +Set up the virtual environment: + +```sh +python3 -m venv .venv +source .venv/bin/activate +pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu # or cu121 for NVIDIA GPU +pip install -r src/lib/video/requirements.txt +``` + +The YOLOv8 model weights (`yolov8n.pt`) are downloaded automatically on first run. + +Then compile and run: + +```sh +lfc src/PandaDemoCamCtrl.lf +bin/PandaDemoCamCtrl.py +``` + +Press `Backspace` to reset the robot arm, `q` to quit. From 4852eed7e4bd793d920ad5a8031cb51fedd2f7e1 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad Date: Thu, 7 May 2026 11:27:12 +0100 Subject: [PATCH 10/10] Minor fixes on documentation --- src/lib/video/README.md | 2 +- src/lib/video/Video.lf | 2 +- src/lib/video/YOLOv8_Webcam.lf | 8 +++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/lib/video/README.md b/src/lib/video/README.md index e5a2d79..56995a5 100644 --- a/src/lib/video/README.md +++ b/src/lib/video/README.md @@ -1,6 +1,6 @@ # Video and YOLO Library -This library provides video capture, display, and DNN-based object recognition using YOLOv5 for use in the `PandaDemoCamCtrl.lf` programs. +This library provides video capture, display, and DNN-based object recognition using YOLOv8 for use in the `PandaDemoCamCtrl.lf` programs. ## Setup diff --git a/src/lib/video/Video.lf b/src/lib/video/Video.lf index 78a95f8..172e28e 100644 --- a/src/lib/video/Video.lf +++ b/src/lib/video/Video.lf @@ -48,7 +48,7 @@ reactor WebCam(webcam_id=0, url="", offset = 0 s, period = 100 ms) { # self.stream.set(cv2.CAP_PROP_FPS, 30) # Set the camera's FPS to 30 =} - reaction(camera_tick) -> camera_frame {=# Minimal buffering to reduce latency. + reaction(camera_tick) -> camera_frame {= # read() is a combination of grab() and retrieve(). ret, frame = self.stream.read() if ret: diff --git a/src/lib/video/YOLOv8_Webcam.lf b/src/lib/video/YOLOv8_Webcam.lf index ef13d59..34c9919 100644 --- a/src/lib/video/YOLOv8_Webcam.lf +++ b/src/lib/video/YOLOv8_Webcam.lf @@ -1,14 +1,12 @@ /** - * This a variant of from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf, but using YOLOv8 instead of YOLOv5. The code is adapted from: - * https://towardsdatascience.com/implementing-real-time-object-detection-system-using-pytorch-and-opencv-70bac41148f7 + * Adapted from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf + * Uses YOLOv8 (ultralytics) instead of YOLOv5. * * Original license: [BSD 2-Clause License] * See README.md for instructions on how to install the required libraries. * Date copied: 2026-05-01 * - * Please see README.md for instructions. This uses ultralytics/yolov8. Adapted from: - * - * One modification from the original: + * Modification from the original: * 1. Removed the deadline on the DNN, for latency purposes. **/