store mic audio with toggle

README.md

@@ -103,7 +103,7 @@ By default, openpilot uploads the driving data to our servers. You can also acce
 openpilot is open source software: the user is free to disable data collection if they wish to do so.
 
 openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
+The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
 
 By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
 </details>

cereal/log.capnp

@@ -2470,7 +2470,7 @@ struct DebugAlert {
 struct UserFlag {
 }
 
-struct Microphone {
+struct SoundPressure @0xdc24138990726023 {
   soundPressure @0 :Float32;
 
   # uncalibrated, A-weighted
@@ -2479,6 +2479,11 @@ struct Microphone {
   filteredSoundPressureWeightedDb @2 :Float32;
 }
 
+struct AudioData {
+  data @0 :Data;
+  sampleRate @1 :UInt32;
+}
+
 struct Touch {
   sec @0 :Int64;
   usec @1 :Int64;
@@ -2556,7 +2561,8 @@ struct Event {
     livestreamDriverEncodeIdx @119 :EncodeIndex;
 
     # microphone data
-    microphone @103 :Microphone;
+    soundPressure @103 :SoundPressure;
+    rawAudioData @147 :AudioData;
 
     # systems stuff
     androidLog @20 :AndroidLogEntry;

cereal/services.py

@@ -73,7 +73,8 @@ def __init__(self, should_log: bool, frequency: float, decimation: Optional[int]
   "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
   "userFlag": (True, 0., 1),
-  "microphone": (True, 10., 10),
+  "soundPressure": (True, 10., 10),
+  "rawAudioData": (False, 20.),
 
   # debug
   "uiDebug": (True, 0., 1),

common/params_keys.h

@@ -99,6 +99,7 @@ inline static std::unordered_map<std::string, uint32_t> keys = {
     {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"PandaSignatures", CLEAR_ON_MANAGER_START},
     {"PrimeType", PERSISTENT},
+    {"RecordAudio", PERSISTENT},
     {"RecordFront", PERSISTENT},
     {"RecordFrontLock", PERSISTENT},  // for the internal fleet
     {"SecOCKey", PERSISTENT | DONT_LOG},

selfdrive/ui/layouts/settings/toggles.py

@@ -21,6 +21,7 @@
   "AlwaysOnDM": "Enable driver monitoring even when openpilot is not engaged.",
   'RecordFront': "Upload data from the driver facing camera and help improve the driver monitoring algorithm.",
   "IsMetric": "Display speed in km/h instead of mph.",
+  "RecordAudio": "Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect.",
 }
 
 
@@ -76,6 +77,12 @@ def __init__(self):
       toggle_item(
         "Use Metric System", DESCRIPTIONS["IsMetric"], self._params.get_bool("IsMetric"), icon="monitoring.png"
       ),
+      toggle_item(
+        "Record Microphone Audio",
+        DESCRIPTIONS["RecordAudio"],
+        self._params.get_bool("RecordAudio"),
+        icon="microphone.png",
+      ),
     ]
 
     self._list_widget = ListView(items)

selfdrive/ui/qt/offroad/settings.cc

@@ -68,6 +68,13 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) {
       "../assets/icons/metric.png",
       false,
     },
+    {
+      "RecordAudio",
+      tr("Record Microphone Audio"),
+      tr("Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect."),
+      "../assets/icons/microphone.png",
+      true,
+    },
   };
 
 

selfdrive/ui/qt/sidebar.cc

@@ -24,10 +24,11 @@ void Sidebar::drawMetric(QPainter &p, const QPair<QString, QString> &label, QCol
   p.drawText(rect.adjusted(22, 0, 0, 0), Qt::AlignCenter, label.first + "\n" + label.second);
 }
 
-Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false) {
+Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false), mic_indicator_pressed(false) {
   home_img = loadPixmap("../assets/images/button_home.png", home_btn.size());
   flag_img = loadPixmap("../assets/images/button_flag.png", home_btn.size());
   settings_img = loadPixmap("../assets/images/button_settings.png", settings_btn.size(), Qt::IgnoreAspectRatio);
+  mic_img = loadPixmap("../assets/icons/microphone.png", QSize(30, 30));
 
   connect(this, &Sidebar::valueChanged, [=] { update(); });
 
@@ -47,12 +48,15 @@ void Sidebar::mousePressEvent(QMouseEvent *event) {
   } else if (settings_btn.contains(event->pos())) {
     settings_pressed = true;
     update();
+  } else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
+    mic_indicator_pressed = true;
+    update();
   }
 }
 
 void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
-  if (flag_pressed || settings_pressed) {
-    flag_pressed = settings_pressed = false;
+  if (flag_pressed || settings_pressed || mic_indicator_pressed) {
+    flag_pressed = settings_pressed = mic_indicator_pressed = false;
     update();
   }
   if (onroad && home_btn.contains(event->pos())) {
@@ -61,6 +65,8 @@ void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
     pm->send("userFlag", msg);
   } else if (settings_btn.contains(event->pos())) {
     emit openSettings();
+  } else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
+    emit openSettings(2, "RecordAudio");
   }
 }
 
@@ -106,6 +112,8 @@ void Sidebar::updateState(const UIState &s) {
     pandaStatus = {{tr("NO"), tr("PANDA")}, danger_color};
   }
   setProperty("pandaStatus", QVariant::fromValue(pandaStatus));
+
+  setProperty("recordingAudio", s.scene.recording_audio);
 }
 
 void Sidebar::paintEvent(QPaintEvent *event) {
@@ -120,6 +128,14 @@ void Sidebar::paintEvent(QPaintEvent *event) {
   p.drawPixmap(settings_btn.x(), settings_btn.y(), settings_img);
   p.setOpacity(onroad && flag_pressed ? 0.65 : 1.0);
   p.drawPixmap(home_btn.x(), home_btn.y(), onroad ? flag_img : home_img);
+  if (recording_audio) {
+    p.setBrush(danger_color);
+    p.setOpacity(mic_indicator_pressed ? 0.65 : 1.0);
+    p.drawRoundedRect(mic_indicator_btn, mic_indicator_btn.height() / 2, mic_indicator_btn.height() / 2);
+    int icon_x = mic_indicator_btn.x() + (mic_indicator_btn.width() - mic_img.width()) / 2;
+    int icon_y = mic_indicator_btn.y() + (mic_indicator_btn.height() - mic_img.height()) / 2;
+    p.drawPixmap(icon_x, icon_y, mic_img);
+  }
   p.setOpacity(1.0);
 
   // network

selfdrive/ui/qt/sidebar.h

@@ -18,6 +18,7 @@ class Sidebar : public QFrame {
   Q_PROPERTY(ItemStatus tempStatus MEMBER temp_status NOTIFY valueChanged);
   Q_PROPERTY(QString netType MEMBER net_type NOTIFY valueChanged);
   Q_PROPERTY(int netStrength MEMBER net_strength NOTIFY valueChanged);
+  Q_PROPERTY(bool recordingAudio MEMBER recording_audio NOTIFY valueChanged);
 
 public:
   explicit Sidebar(QWidget* parent = 0);
@@ -36,8 +37,8 @@ public slots:
   void mouseReleaseEvent(QMouseEvent *event) override;
   void drawMetric(QPainter &p, const QPair<QString, QString> &label, QColor c, int y);
 
-  QPixmap home_img, flag_img, settings_img;
-  bool onroad, flag_pressed, settings_pressed;
+  QPixmap home_img, flag_img, settings_img, mic_img;
+  bool onroad, recording_audio, flag_pressed, settings_pressed, mic_indicator_pressed;
   const QMap<cereal::DeviceState::NetworkType, QString> network_type = {
     {cereal::DeviceState::NetworkType::NONE, tr("--")},
     {cereal::DeviceState::NetworkType::WIFI, tr("Wi-Fi")},
@@ -50,6 +51,7 @@ public slots:
 
   const QRect home_btn = QRect(60, 860, 180, 180);
   const QRect settings_btn = QRect(50, 35, 200, 117);
+  const QRect mic_indicator_btn = QRect(158, 252, 75, 40);
   const QColor good_color = QColor(255, 255, 255);
   const QColor warning_color = QColor(218, 202, 37);
   const QColor danger_color = QColor(201, 34, 49);

selfdrive/ui/soundd.py

@@ -135,7 +135,7 @@ def soundd_thread(self):
     # sounddevice must be imported after forking processes
     import sounddevice as sd
 
-    sm = messaging.SubMaster(['selfdriveState', 'microphone'])
+    sm = messaging.SubMaster(['selfdriveState', 'soundPressure'])
 
     with self.get_stream(sd) as stream:
       rk = Ratekeeper(20)
@@ -144,8 +144,8 @@ def soundd_thread(self):
       while True:
         sm.update(0)
 
-        if sm.updated['microphone'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
-          self.spl_filter_weighted.update(sm["microphone"].soundPressureWeightedDb)
+        if sm.updated['soundPressure'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
+          self.spl_filter_weighted.update(sm["soundPressure"].soundPressureWeightedDb)
           self.current_volume = self.calculate_volume(float(self.spl_filter_weighted.x))
 
         self.get_audible_alert(sm)

selfdrive/ui/ui.cc

@@ -60,6 +60,9 @@ static void update_state(UIState *s) {
     scene.light_sensor = -1;
   }
   scene.started = sm["deviceState"].getDeviceState().getStarted() && scene.ignition;
+
+  auto params = Params();
+  scene.recording_audio = params.getBool("RecordAudio") && scene.started;
 }
 
 void ui_update_params(UIState *s) {

selfdrive/ui/ui.h

@@ -58,7 +58,7 @@ typedef struct UIScene {
   cereal::LongitudinalPersonality personality;
 
   float light_sensor = -1;
-  bool started, ignition, is_metric;
+  bool started, ignition, is_metric, recording_audio;
   uint64_t started_frame;
 } UIScene;
 

system/loggerd/loggerd.cc

@@ -226,19 +226,21 @@ void loggerd_thread() {
   for (const auto& [_, it] : services) {
     const bool encoder = util::ends_with(it.name, "EncodeData");
     const bool livestream_encoder = util::starts_with(it.name, "livestream");
-    if (!it.should_log && (!encoder || livestream_encoder)) continue;
-    LOGD("logging %s", it.name.c_str());
-
-    SubSocket * sock = SubSocket::create(ctx.get(), it.name);
-    assert(sock != NULL);
-    poller->registerSocket(sock);
-    service_state[sock] = {
-      .name = it.name,
-      .counter = 0,
-      .freq = it.decimation,
-      .encoder = encoder,
-      .user_flag = it.name == "userFlag",
-    };
+    const bool record_audio = (it.name == "rawAudioData") && Params().getBool("RecordAudio");
+    if (it.should_log || (encoder && !livestream_encoder) || record_audio) {
+      LOGD("logging %s", it.name.c_str());
+
+      SubSocket * sock = SubSocket::create(ctx.get(), it.name);
+      assert(sock != NULL);
+      poller->registerSocket(sock);
+      service_state[sock] = {
+        .name = it.name,
+        .counter = 0,
+        .freq = it.decimation,
+        .encoder = encoder,
+        .user_flag = it.name == "userFlag",
+      };
+    }
   }
 
   LoggerdState s;

system/micd.py

@@ -9,10 +9,10 @@
 from openpilot.common.swaglog import cloudlog
 
 RATE = 10
-FFT_SAMPLES = 4096
+FFT_SAMPLES = 1600 # 100ms
 REFERENCE_SPL = 2e-5  # newtons/m^2
-SAMPLE_RATE = 44100
-SAMPLE_BUFFER = 4096  # approx 100ms
+SAMPLE_RATE = 16000
+SAMPLE_BUFFER = 800  # 50ms
 
 
 @cache
@@ -45,7 +45,7 @@ def apply_a_weighting(measurements: np.ndarray) -> np.ndarray:
 class Mic:
   def __init__(self):
     self.rk = Ratekeeper(RATE)
-    self.pm = messaging.PubMaster(['microphone'])
+    self.pm = messaging.PubMaster(['soundPressure', 'rawAudioData'])
 
     self.measurements = np.empty(0)
 
@@ -61,12 +61,12 @@ def update(self):
       sound_pressure_weighted = self.sound_pressure_weighted
       sound_pressure_level_weighted = self.sound_pressure_level_weighted
 
-    msg = messaging.new_message('microphone', valid=True)
-    msg.microphone.soundPressure = float(sound_pressure)
-    msg.microphone.soundPressureWeighted = float(sound_pressure_weighted)
-    msg.microphone.soundPressureWeightedDb = float(sound_pressure_level_weighted)
+    msg = messaging.new_message('soundPressure', valid=True)
+    msg.soundPressure.soundPressure = float(sound_pressure)
+    msg.soundPressure.soundPressureWeighted = float(sound_pressure_weighted)
+    msg.soundPressure.soundPressureWeightedDb = float(sound_pressure_level_weighted)
 
-    self.pm.send('microphone', msg)
+    self.pm.send('soundPressure', msg)
     self.rk.keep_time()
 
   def callback(self, indata, frames, time, status):
@@ -76,6 +76,12 @@ def callback(self, indata, frames, time, status):
 
     Logged A-weighted equivalents are rough approximations of the human-perceived loudness.
     """
+    msg = messaging.new_message('rawAudioData', valid=True)
+    audio_data_int_16 = (indata[:, 0] * 32767).astype(np.int16)
+    msg.rawAudioData.data = audio_data_int_16.tobytes()
+    msg.rawAudioData.sampleRate = SAMPLE_RATE
+    self.pm.send('rawAudioData', msg)
+
     with self.lock:
       self.measurements = np.concatenate((self.measurements, indata[:, 0]))
 

tools/scripts/extract_audio.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import os
+import sys
+import wave
+import argparse
+import numpy as np
+
+from openpilot.tools.lib.logreader import LogReader, ReadMode
+
+
+def extract_audio(route_or_segment_name, output_file=None, play=False):
+  lr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE)
+  audio_messages = list(lr.filter("rawAudioData"))
+  if not audio_messages:
+    print("No rawAudioData messages found in logs")
+    return
+  sample_rate = audio_messages[0].sampleRate
+
+  audio_chunks = []
+  total_frames = 0
+  for msg in audio_messages:
+    audio_array = np.frombuffer(msg.data, dtype=np.int16)
+    audio_chunks.append(audio_array)
+    total_frames += len(audio_array)
+  full_audio = np.concatenate(audio_chunks)
+
+  print(f"Found {total_frames} frames from {len(audio_messages)} audio messages at {sample_rate} Hz")
+
+  if output_file:
+    if write_wav_file(output_file, full_audio, sample_rate):
+      print(f"Audio written to {output_file}")
+    else:
+      print("Audio extraction canceled.")
+  if play:
+    play_audio(full_audio, sample_rate)
+
+
+def write_wav_file(filename, audio_data, sample_rate):
+  if os.path.exists(filename):
+    if input(f"File '{filename}' exists. Overwrite? (y/N): ").lower() not in ['y', 'yes']:
+      return False
+
+  with wave.open(filename, 'wb') as wav_file:
+    wav_file.setnchannels(1)  # Mono
+    wav_file.setsampwidth(2)  # 16-bit
+    wav_file.setframerate(sample_rate)
+    wav_file.writeframes(audio_data.tobytes())
+  return True
+
+
+def play_audio(audio_data, sample_rate):
+  try:
+    import sounddevice as sd
+
+    print("Playing audio... Press Ctrl+C to stop")
+    sd.play(audio_data, sample_rate)
+    sd.wait()
+  except KeyboardInterrupt:
+    print("\nPlayback stopped")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Extract audio data from openpilot logs")
+  parser.add_argument("-o", "--output", help="Output WAV file path")
+  parser.add_argument("--play", action="store_true", help="Play audio with sounddevice")
+  parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name")
+
+  if len(sys.argv) == 1:
+    parser.print_help()
+    sys.exit()
+  args = parser.parse_args()
+
+  output_file = args.output
+  if not args.output and not args.play:
+    output_file = "extracted_audio.wav"
+
+  extract_audio(args.route_or_segment_name.strip(), output_file, args.play)