CANParser: remove dependence on cereal (commaai#934)

* move car interfaces to opendbc

* Revert "move car interfaces to opendbc"

This reverts commit 0589ea0.

* remove dependence on cereal

* parse can strings in a c++ helper function

* remove all cereal

* reserve

* use itervalues

* rebase master

* cleanup

* keep the same style

* revert unrelated changes

* truly no cereal!

* same capitalization

same capitalization

* parser: declutter nanos usage (commaai#1067)

thought there'd be more

* revert more

---------

Co-authored-by: Shane Smiskol <[email protected]>

Dockerfile

@@ -44,11 +44,6 @@ ENV PYTHONPATH=/project
 RUN git config --global --add safe.directory '*'
 
 WORKDIR /project
-RUN git clone https://github.com/commaai/cereal.git /project/cereal && \
-    cd /project/cereal && \
-    git checkout 861144c136c91f70dcbc652c2ffe99f57440ad47 && \
-    rm -rf .git && \
-    scons -j$(nproc) --minimal
 
 COPY SConstruct .
 COPY ./site_scons /project/site_scons

SConstruct

@@ -6,8 +6,6 @@ import numpy as np
 zmq = 'zmq'
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 
-cereal_dir = Dir('.')
-
 python_path = sysconfig.get_paths()['include']
 cpppath = [
   '#',
@@ -56,11 +54,6 @@ env = Environment(
 common = ''
 Export('env', 'zmq', 'arch', 'common')
 
-cereal = [File('#cereal/libcereal.a')]
-messaging = [File('#cereal/libmessaging.a')]
-Export('cereal', 'messaging')
-
-
 envCython = env.Clone()
 envCython["CPPPATH"] += [np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
@@ -80,6 +73,4 @@ envCython["LIBS"] = python_libs
 
 Export('envCython')
 
-
-SConscript(['cereal/SConscript'])
 SConscript(['opendbc/can/SConscript'])

can/SConscript

@@ -1,12 +1,12 @@
-Import('env', 'envCython', 'cereal', 'common', 'arch')
+Import('env', 'envCython', 'common', 'arch')
 
 import os
 
 envDBC = env.Clone()
 dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("..").abspath)
 envDBC['CXXFLAGS'] += [dbc_file_path]
 src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]
-libs = [common, "capnp", "kj", "zmq"]
+libs = [common, "zmq"]
 
 # shared library for openpilot
 LINKFLAGS = envDBC["LINKFLAGS"]

can/common.h

@@ -6,13 +6,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include <capnp/dynamic.h>
-#include <capnp/serialize.h>
-
-#ifndef DYNAMIC_CAPNP
-#include "cereal/gen/cpp/log.capnp.h"
-#endif
-
 #include "opendbc/can/logger.h"
 #include "opendbc/can/common_dbc.h"
 
@@ -34,6 +27,17 @@ unsigned int xor_checksum(uint32_t address, const Signal &sig, const std::vector
 unsigned int hkg_can_fd_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 
+struct CanFrame {
+  long src;
+  uint32_t address;
+  std::vector<uint8_t> dat;
+};
+
+struct CanData {
+  uint64_t nanos;
+  std::vector<CanFrame> frames;
+};
+
 class MessageState {
 public:
   std::string name;
@@ -60,8 +64,6 @@ class MessageState {
 class CANParser {
 private:
   const int bus;
-  kj::Array<capnp::word> aligned_buf;
-
   const DBC *dbc = NULL;
   std::unordered_map<uint32_t, MessageState> message_states;
 
@@ -77,14 +79,12 @@ class CANParser {
   CANParser(int abus, const std::string& dbc_name,
             const std::vector<std::pair<uint32_t, int>> &messages);
   CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
-  #ifndef DYNAMIC_CAPNP
-  void update_string(const std::string &data, bool sendcan);
-  void update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan);
-  void UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans);
-  #endif
-  void UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cans);
-  void UpdateValid(uint64_t nanos);
+  void update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals);
   void query_latest(std::vector<SignalValue> &vals, uint64_t last_ts = 0);
+
+protected:
+  void UpdateCans(const CanData &can);
+  void UpdateValid(uint64_t nanos);
 };
 
 class CANPacker {

can/common.pxd

@@ -67,11 +67,20 @@ cdef extern from "common_dbc.h":
 cdef extern from "common.h":
   cdef const DBC* dbc_lookup(const string) except +
 
+  cdef struct CanFrame:
+    long src
+    uint32_t address
+    vector[uint8_t] dat
+
+  cdef struct CanData:
+    uint64_t nanos
+    vector[CanFrame] frames
+
   cdef cppclass CANParser:
     bool can_valid
     bool bus_timeout
     CANParser(int, string, vector[pair[uint32_t, int]]) except +
-    void update_strings(vector[string]&, vector[SignalValue]&, bool) except +
+    void update(vector[CanData]&, vector[SignalValue]&) except +
 
   cdef cppclass CANPacker:
    CANPacker(string)

can/parser.cc

@@ -93,7 +93,7 @@ bool MessageState::update_counter_generic(int64_t v, int cnt_size) {
 
 
 CANParser::CANParser(int abus, const std::string& dbc_name, const std::vector<std::pair<uint32_t, int>> &messages)
-  : bus(abus), aligned_buf(kj::heapArray<capnp::word>(1024)) {
+  : bus(abus) {
   dbc = dbc_lookup(dbc_name);
   assert(dbc);
 
@@ -157,64 +157,42 @@ CANParser::CANParser(int abus, const std::string& dbc_name, bool ignore_checksum
   }
 }
 
-#ifndef DYNAMIC_CAPNP
-void CANParser::update_string(const std::string &data, bool sendcan) {
-  // format for board, make copy due to alignment issues.
-  const size_t buf_size = (data.length() / sizeof(capnp::word)) + 1;
-  if (aligned_buf.size() < buf_size) {
-    aligned_buf = kj::heapArray<capnp::word>(buf_size);
-  }
-  memcpy(aligned_buf.begin(), data.data(), data.length());
-
-  // extract the messages
-  capnp::FlatArrayMessageReader cmsg(aligned_buf.slice(0, buf_size));
-  cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
-  if (first_nanos == 0) {
-    first_nanos = event.getLogMonoTime();
-  }
-  last_nanos = event.getLogMonoTime();
-
-  auto cans = sendcan ? event.getSendcan() : event.getCan();
-  UpdateCans(last_nanos, cans);
-
-  UpdateValid(last_nanos);
-}
-
-void CANParser::update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan) {
+void CANParser::update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals) {
   uint64_t current_nanos = 0;
-  for (const auto &d : data) {
-    update_string(d, sendcan);
+  for (const auto &c : can_data) {
+    if (first_nanos == 0) {
+      first_nanos = c.nanos;
+    }
     if (current_nanos == 0) {
-      current_nanos = last_nanos;
+      current_nanos = c.nanos;
     }
+    last_nanos = c.nanos;
+
+    UpdateCans(c);
+    UpdateValid(last_nanos);
   }
   query_latest(vals, current_nanos);
 }
 
-void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans) {
-  //DEBUG("got %d messages\n", cans.size());
+void CANParser::UpdateCans(const CanData &can) {
+  //DEBUG("got %zu messages\n", can.frames.size());
 
   bool bus_empty = true;
 
-  // parse the messages
-  for (const auto cmsg : cans) {
-    if (cmsg.getSrc() != bus) {
+  for (const auto &frame : can.frames) {
+    if (frame.src != bus) {
       // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
       continue;
     }
     bus_empty = false;
 
-    auto state_it = message_states.find(cmsg.getAddress());
+    auto state_it = message_states.find(frame.address);
     if (state_it == message_states.end()) {
       // DEBUG("skip %d: not specified\n", cmsg.getAddress());
       continue;
     }
-
-    auto dat = cmsg.getDat();
-
-    if (dat.size() > 64) {
-      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", cmsg.getAddress(), dat.size());
+    if (frame.dat.size() > 64) {
+      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", frame.address, frame.dat.size());
       continue;
     }
 
@@ -224,42 +202,14 @@ void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::R
     //  continue;
     //}
 
-    // TODO: can remove when we ignore unexpected can msg lengths
-    // make sure the data_size is not less than state_it->second.size
-    size_t data_size = std::max<size_t>(dat.size(), state_it->second.size);
-    std::vector<uint8_t> data(data_size, 0);
-    memcpy(data.data(), dat.begin(), dat.size());
-    state_it->second.parse(nanos, data);
+    state_it->second.parse(can.nanos, frame.dat);
   }
 
   // update bus timeout
   if (!bus_empty) {
-    last_nonempty_nanos = nanos;
-  }
-  bus_timeout = (nanos - last_nonempty_nanos) > bus_timeout_threshold;
-}
-#endif
-
-void CANParser::UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cmsg) {
-  // assume message struct is `cereal::CanData` and parse
-  assert(cmsg.has("address") && cmsg.has("src") && cmsg.has("dat"));
-
-  if (cmsg.get("src").as<uint8_t>() != bus) {
-    DEBUG("skip %d: wrong bus\n", cmsg.get("address").as<uint32_t>());
-    return;
+    last_nonempty_nanos = can.nanos;
   }
-
-  auto state_it = message_states.find(cmsg.get("address").as<uint32_t>());
-  if (state_it == message_states.end()) {
-    DEBUG("skip %d: not specified\n", cmsg.get("address").as<uint32_t>());
-    return;
-  }
-
-  auto dat = cmsg.get("dat").as<capnp::Data>();
-  if (dat.size() > 64) return; // shouldn't ever happen
-  std::vector<uint8_t> data(dat.size(), 0);
-  memcpy(data.data(), dat.begin(), dat.size());
-  state_it->second.parse(nanos, data);
+  bus_timeout = (can.nanos - last_nonempty_nanos) > bus_timeout_threshold;
 }
 
 void CANParser::UpdateValid(uint64_t nanos) {

can/parser_pyx.pyx

@@ -8,7 +8,7 @@ from libcpp.vector cimport vector
 from libc.stdint cimport uint32_t
 
 from .common cimport CANParser as cpp_CANParser
-from .common cimport dbc_lookup, SignalValue, DBC
+from .common cimport dbc_lookup, SignalValue, DBC, CanData, CanFrame
 
 import numbers
 from collections import defaultdict
@@ -65,17 +65,42 @@ cdef class CANParser:
       del self.can
 
   def update_strings(self, strings, sendcan=False):
+    # input format:
+    # [nanos, [[address, data, src], ...]]
+    # [[nanos, [[address, data, src], ...], ...]]
     for address in self.addresses:
       self.vl_all[address].clear()
 
-    cdef vector[SignalValue] new_vals
     cur_address = -1
     vl = {}
     vl_all = {}
     ts_nanos = {}
     updated_addrs = set()
 
-    self.can.update_strings(strings, new_vals, sendcan)
+    cdef vector[SignalValue] new_vals
+    cdef CanFrame* frame
+    cdef CanData* can_data
+    cdef vector[CanData] can_data_array
+
+    try:
+      if len(strings) and not isinstance(strings[0], (list, tuple)):
+        strings = [strings]
+
+      can_data_array.reserve(len(strings))
+      for s in strings:
+        can_data = &(can_data_array.emplace_back())
+        can_data.nanos = s[0]
+        can_data.frames.reserve(len(s[1]))
+        for f in s[1]:
+          frame = &(can_data.frames.emplace_back())
+          frame.address = f[0]
+          frame.dat = f[1]
+          frame.src = f[2]
+    except TypeError:
+      raise RuntimeError("invalid parameter")
+
+    self.can.update(can_data_array, new_vals)
+
     cdef vector[SignalValue].iterator it = new_vals.begin()
     cdef SignalValue* cv
     while it != new_vals.end():

can/tests/test_checksums.py

@@ -1,6 +1,5 @@
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
-from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp
 
 
 class TestCanChecksums:
@@ -28,8 +27,7 @@ def test_honda_checksum(self):
         packer.make_can_msg("LKAS_HUD", 0, values),
         packer.make_can_msg("LKAS_HUD_A", 0, values),
       ]
-      can_strings = [can_list_to_can_capnp(msgs), ]
-      parser.update_strings(can_strings)
+      parser.update_strings([0, msgs])
 
       assert parser.vl['LKAS_HUD']['CHECKSUM'] == std
       assert parser.vl['LKAS_HUD_A']['CHECKSUM'] == ext