Add camera-controlled Panda demo with YOLOv8 person detection

.gitignore

@@ -5,3 +5,5 @@ src-gen
 /include
 fed-gen
 MUJOCO_LOG.TXT
+.venv
+*.pt

README.md

@@ -29,3 +29,28 @@ Build the demos using `lfc` or `make`:
 * [MuJoCoCarDemo](src/MuJoCoCarDemo.lf): Simple drivable car.
 * [MuJoCoCarAutoDemo](src/MuJoCoCarAutoDemo.lf): Simple drivable car.
 * [PandaDemo](src/PandaDemo.lf): Franka Emika Panda robot doing gyrations.
+* [PandaDemoCamCtrl](src/PandaDemoCamCtrl.lf): Franka Emika Panda robot with camera-based person detection. The robot pauses automatically when two or more people are detected by a YOLOv8 DNN monitoring the webcam feed, and resumes when the area is clear.
+
+### PandaDemoCamCtrl Prerequisites
+
+This demo uses [YOLOv8](https://github.com/ultralytics/ultralytics) and OpenCV for real-time object detection. It requires a Python virtual environment with the dependencies listed in [src/lib/video/requirements.txt](src/lib/video/requirements.txt), which covers all reactors under [src/lib/video/](src/lib/video/).
+
+Set up the virtual environment:
+
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu  # or cu121 for NVIDIA GPU
+pip install -r src/lib/video/requirements.txt
+```
+
+The YOLOv8 model weights (`yolov8n.pt`) are downloaded automatically on first run.
+
+Then compile and run:
+
+```sh
+lfc src/PandaDemoCamCtrl.lf
+bin/PandaDemoCamCtrl.py
+```
+
+Press `Backspace` to reset the robot arm, `q` to quit.

src/PandaDemoCamCtrl.lf

@@ -0,0 +1,162 @@
+/**
+ * Franka Emika Panda robot demo with camera-based person detection.
+ *
+ * Moves each of the eight joints of the Panda robot through a range of values defined in a CSV
+ * file, while using a YOLOv8 DNN to monitor the webcam feed. The robot pauses automatically
+ * when two or more people are detected nearby, and resumes when the area is clear.
+ *
+ * See [README.md](../README.md) for prerequisites and installation instructions.
+ *
+ * The CSV file "PandaDemoParameters.csv" (located in the same directory) defines motion
+ * parameters. There are three datasets corresponding to three different speeds of motion.
+ * To select a dataset, change the top-level `dataset` parameter to 0, 1, or 2.
+ * 
+ * The Robot arm will stop moving when the number of people detected by the DNN is greater than 
+ * or equal to 2. The robot will resume moving when the number of people detected is less than 2.
+ *
+ * The demo is based on the original Panda demo, as well as Yolo, DNN and Video examples from the
+ * LF playground.
+ *
+ * @author Chadlia Jerad
+ */
+
+target Python {
+  single-threaded: true,
+  keepalive: true
+}
+
+import MuJoCoPanda from "lib/MuJoCoPanda.lf"
+import DNN, Plotter from "lib/video/YOLOv8_Webcam.lf"
+import WebCam, Display from "lib/video/Video.lf"
+
+preamble {=
+  import csv
+  from mujoco.glfw import glfw
+=}
+
+reactor SimpleController(
+    period = 100 ms,
+    dataset = 0,
+    step=0.1,
+    high_limit=1.0,
+    low_limit=-1.0,
+    bank_index=0,
+    parameter_file="") 
+{
+  timer t(0, period)
+  input restart
+  input pause_play
+  output command
+  state motor = 0.0
+
+  # In Python, can't overwrite parameter values, so we use state variables instead.
+  state step_value = step
+  state high_limit_value = high_limit
+  state low_limit_value = low_limit
+
+  reaction(startup) {=
+    if self.parameter_file:
+      with open(self.parameter_file, "r", encoding="utf-8-sig") as f:
+        rows = list(csv.reader(f))
+      row_idx = (9 * self.dataset) + self.bank_index + 1
+      if row_idx < len(rows):
+        row = rows[row_idx]
+        self.step_value = float(row[0].strip())
+        self.high_limit_value = float(row[1].strip())
+        self.low_limit_value = float(row[2].strip())
+  =}
+
+  initial mode PLAY {
+    reaction(restart) {=
+      self.motor = 0.0
+    =}
+
+    reaction(pause_play) -> PAUSE {=
+      PAUSE.set()
+    =}
+
+    reaction(t) -> command {=
+      command.set(self.motor)
+      self.motor += self.step_value
+      if self.motor > self.high_limit_value or self.motor < self.low_limit_value:
+        self.step_value = -self.step_value
+    =}
+  } 
+
+  mode PAUSE {
+    reaction(t) -> command {=
+      command.set(0.0)
+    =}
+
+    reaction(pause_play) -> reset(PLAY) {=
+      PLAY.set()
+    =}
+  }
+}
+
+main reactor(dataset = 3) {
+  timer read_state_timer(0, 500 ms)
+  state nbr_person_detected = 0
+  state is_paused = False
+
+  m = new MuJoCoPanda(frame_period = 50 ms)
+  c = new[8] SimpleController(
+      period = 100 ms,
+      dataset = dataset,
+      parameter_file = {= lf.source_directory() + "/PandaDemoParameters.csv" =}
+  )
+  # Offset prevents deadline violations during startup.
+  webcam = new WebCam(webcam_id=0, offset = 3 s) 
+  // webcam = new WebCam(url="http://10.152.67.28:8080/video", offset = 3 s)
+  dnn = new DNN()
+  plotter = new Plotter(label_deadline = 100 msec)
+  display = new Display()
+
+  c.command -> m.motor
+  (webcam.camera_frame)+ -> dnn.frame, plotter.frame
+  dnn.labels, dnn.label_coordinates -> plotter.labels, plotter.label_coordinates
+
+  dnn.model -> plotter.model
+  plotter.result -> display.frame
+
+  reaction(startup) {=
+    print("*** Backspace to reset.")
+    print("*** Type q to quit.\n")
+  =}
+
+  reaction(m.key) -> m.restart, c.restart {=
+    if m.key.value.act == glfw.PRESS:
+      if m.key.value.key == glfw.KEY_BACKSPACE:
+        m.restart.set(True)
+        for i in range(8):
+          c[i].restart.set(True)
+      elif m.key.value.key == glfw.KEY_Q:
+        lf.request_stop()
+  =}
+
+  reaction(read_state_timer) -> m.read_state {=
+    m.read_state.set(True)
+  =}
+
+  reaction(dnn.labels) -> c.pause_play{=
+    person_count = sum(1 for label in dnn.labels.value if int(label) == 0)
+    self.nbr_person_detected = person_count
+    should_pause = person_count >= 2
+    if should_pause and not self.is_paused:
+      print(f"{person_count} persons detected at {lf.time.logical_elapsed()/1000000000} => Pausing robot: too many people around!")
+      for i in range(8):
+        c[i].pause_play.set(True)
+      self.is_paused = True
+    elif not should_pause and self.is_paused:
+      print(f"{person_count} persons detected at {lf.time.logical_elapsed()/1000000000} => Resuming robot.")
+      for i in range(8):
+        c[i].pause_play.set(True)
+      self.is_paused = False
+  =}
+
+  reaction(m.joint_pos, m.joint_vel) {=
+    qpos = " ".join(f"{m.joint_pos[i].value:.4f}" for i in range(9))
+    qvel = " ".join(f"{m.joint_vel[i].value:.4f}" for i in range(9))
+    # print(f"qpos={qpos} | qvel={qvel}")
+  =}
+}

src/PandaDemoParameters.csv

@@ -24,4 +24,13 @@ dataset 2,,
 0.2,3.14159,-3.14159
 0.2,3.5,-1
 0.2,1,-1
-20,255,0
+20,255,0
+dataset 3,,
+0.05,2.5,-2
+0.05,1.2,-0.8
+0.05,1,-1
+0.05,0,-2
+0.05,2,-2
+0.05,2.5,-0.3
+0.05,1,-1
+10,200,0

src/lib/MuJoCoBase.lf

@@ -1,9 +1,11 @@
 /** @file Base class for reactors using the MuJoCo physics-based simulation framework. */
 target Python {
   keepalive: true,
-  cmake-args: {
-    Python_EXECUTABLE: "mjpython"
-  }
+  single-threaded: true,
+  // If you are using `mjpython`, then uncomment the following lines
+  // cmake-args: {
+  //   Python_EXECUTABLE: "mjpython"
+  // }
 }
 
 preamble {=

src/lib/video/README.md

@@ -0,0 +1,48 @@
+# Video and YOLO Library
+
+This library provides video capture, display, and DNN-based object recognition using YOLOv8 for use in the `PandaDemoCamCtrl.lf` programs.
+
+## Setup
+
+### 1. Install PyTorch
+
+Go to the [PyTorch website](https://pytorch.org/get-started/locally/) and follow the instructions for your platform.
+
+**If you have an NVIDIA GPU**, select the correct CUDA version on the installation page and follow the generated command, for example:
+
+```bash
+pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
+```
+
+**If you have no GPU (CPU only)**, install the CPU builds explicitly. Using the generic PyPI packages will cause a version mismatch and a segfault at runtime:
+
+```bash
+pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+```
+
+### 2. Install remaining dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+> **Note:** `requirements.txt` intentionally omits `torch` and `torchvision` so that the correct build variant (CPU or CUDA) can be chosen in the step above.
+
+### 3. Font warning fix (OpenCV / Qt)
+
+If you see repeated `QFontDatabase: Cannot find font directory` warnings, Qt's bundled font directory is missing. Fix it by symlinking system DejaVu fonts into the expected location:
+
+```bash
+mkdir -p "$(python3 -c 'import cv2, os; print(os.path.dirname(cv2.__file__))')/qt/fonts"
+ln -s /usr/share/fonts/truetype/dejavu/*.ttf \
+      "$(python3 -c 'import cv2, os; print(os.path.dirname(cv2.__file__))')/qt/fonts/"
+```
+
+### 4. Compile and run
+
+Compile with `lfc` and run the generated Python program:
+
+```bash
+lfc src/PandaDemoCamCtrl.lf
+bin/PandaDemoCamCtrl
+```

src/lib/video/Video.lf

@@ -0,0 +1,96 @@
+/**
+ * Copied from: https://github.com/lf-lang/playground-lingua-franca/blob/main/examples/Python/src/YOLOv5/Video.lf
+ * Original license: [BSD 2-Clause License]
+ * See README.md for instructions on how to install the required libraries.
+ * Date copied: 2026-05-01
+ * 
+ * Two modifications from the original:
+ * 1. Added support for IP camera streams via the url parameter of WebCam.
+ * 2. Added Minimal buffering to reduce latency.
+**/
+
+/** Video capture and playback example using OpenCV. Please see README.md for instructions. */
+target Python {
+  single-threaded: true  # OpenCV crashes if we use the multithreaded version.
+}
+
+preamble {=
+  import cv2
+=}
+
+/**
+ * Produce a sequence of frames with the specified offset and period.
+ * @param webcam_id The ID of the camera (default 0).
+ * @param url An optional URL to an IP camera stream. If specified, this overrides webcam_id.
+ * @param offset Time until frames start to be captured.
+ * @param period The period with which frames will be read.
+ */
+reactor WebCam(webcam_id=0, url="", offset = 0 s, period = 100 ms) {
+  output camera_frame
+
+  state stream
+  timer camera_tick(offset, period)
+
+  reaction(startup) {=
+    if (self.url != ""):
+      self.stream = cv2.VideoCapture(self.url)
+    else:
+      self.stream = cv2.VideoCapture(self.webcam_id, cv2.CAP_ANY) # or CAP_DSHOW
+
+    # Minimal buffering to reduce latency.
+    self.stream.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+
+    if not self.stream.isOpened():
+      sys.stderr.write("Error: Failed to capture from the webcam.\n")
+      exit(1)
+
+    # Here, LF is in charge of the timing, so do not set the frame rate.
+    # self.stream.set(cv2.CAP_PROP_FPS, 30) # Set the camera's FPS to 30
+  =}
+
+  reaction(camera_tick) -> camera_frame {=
+    # read() is a combination of grab() and retrieve().
+    ret, frame = self.stream.read()
+    if ret:
+      camera_frame.set(frame)
+    else:
+      print("WARNING: Camera frame missing.")
+  =}
+
+  reaction(shutdown) {=
+    self.stream.release()
+  =}
+}
+
+/** Display video frames. */
+reactor Display {
+  input frame
+  state frame_count = 0
+
+  reaction(startup) {=
+    print("\n******* Press 'q' in the video window to exit *******\n")
+  =}
+
+  reaction(frame) {=
+    self.frame_count += 1
+    # Every hundred or so frames, report the frame rate.
+    if (self.frame_count % 100 == 0):
+      print(f"** Average frame rate: {self.frame_count * SEC(1) / lf.time.physical_elapsed()} f/s")
+
+    cv2.imshow("frame", frame.value)
+    # press 'Q' if you want to exit
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+      request_stop()
+  =}
+
+  reaction(shutdown) {=
+    # Destroy the all windows now
+    cv2.destroyAllWindows()
+  =}
+}
+
+main reactor {
+  webcam = new WebCam(webcam_id = 1)
+  display = new Display()
+  webcam.camera_frame -> display.frame
+}