remove all cereal

can/SConscript

@@ -16,9 +16,8 @@ envDBC.Library('libdbc_static', src, LIBS=libs)
 
 # Build packer and parser
 lenv = envCython.Clone()
-lenv.Library('strings_to_candata', ['strings_to_candata.cc'])
 lenv["LINKFLAGS"] += [libdbc[0].get_labspath()]
-parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx', LIBS=['strings_to_candata', 'capnp', 'kj'] + lenv["LIBS"])
+parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx')
 packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx')
 
 lenv.Depends(parser, libdbc)

can/common.h

@@ -30,7 +30,7 @@ unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vect
 
 struct CanFrame {
   long src;
-  long address;
+  uint32_t address;
   std::vector<uint8_t> dat;
 };
 

can/common.pxd

@@ -66,7 +66,7 @@ cdef extern from "common.h":
 
   cdef struct CanFrame:
     long src
-    long address
+    uint32_t address
     vector[uint8_t] dat
 
   cdef struct CanData:

can/parser_pyx.pyx

@@ -5,19 +5,15 @@ from cython.operator cimport dereference as deref, preincrement as preinc
 from libcpp.pair cimport pair
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-from libcpp cimport bool
 from libcpp.unordered_set cimport unordered_set
 from libc.stdint cimport uint32_t
 
 from .common cimport CANParser as cpp_CANParser
-from .common cimport dbc_lookup, SignalValue, DBC, CanData
+from .common cimport dbc_lookup, SignalValue, DBC, CanData, CanFrame
 
 import numbers
 from collections import defaultdict
 
-cdef extern from "strings_to_candata.cc":
-  void strings_to_candata(const vector[string] &strings, vector[CanData] &candata, bool sendcan) except +
-
 cdef class CANParser:
   cdef:
     cpp_CANParser *can
@@ -74,16 +70,38 @@ cdef class CANParser:
       del self.can
 
   def update_strings(self, strings, sendcan=False):
+    # input format:
+    # [nanos, [[address, 0, data, src], ...]]
+    # [[nanos, [[address, 0, data, src], ...], ...]]
+
     for v in self.vl_all.values():
       for l in v.values():  # no-cython-lint
         l.clear()
 
     cdef vector[SignalValue] new_vals
     cdef unordered_set[uint32_t] updated_addrs
-    cdef vector[CanData] can_data
-
-    strings_to_candata(strings, can_data, sendcan)
-    self.can.update(can_data, new_vals)
+    cdef CanFrame* frame
+    cdef CanData* can_data
+    cdef vector[CanData] can_data_array
+
+    try:
+      if len(strings) and not isinstance(strings[0], (list, tuple)):
+        strings = [strings]
+
+      can_data_array.reserve(len(strings))
+      for s in strings:
+        can_data = &(can_data_array.emplace_back())
+        can_data.nanos = s[0]
+        can_data.frames.reserve(len(s[1]))
+        for f in s[1]:
+          frame = &(can_data.frames.emplace_back())
+          frame.address = f[0]
+          frame.dat = f[2]
+          frame.src = f[3]
+    except TypeError:
+      raise RuntimeError("invalid parameter")
+
+    self.can.update(can_data_array, new_vals)
 
     cdef vector[SignalValue].iterator it = new_vals.begin()
     cdef SignalValue* cv

can/tests/test_checksums.py

@@ -3,7 +3,6 @@
 
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
-from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp
 
 
 class TestCanChecksums(unittest.TestCase):
@@ -31,8 +30,7 @@ def test_honda_checksum(self):
         packer.make_can_msg("LKAS_HUD", 0, values),
         packer.make_can_msg("LKAS_HUD_A", 0, values),
       ]
-      can_strings = [can_list_to_can_capnp(msgs), ]
-      parser.update_strings(can_strings)
+      parser.update_strings([0, msgs])
 
       self.assertEqual(parser.vl['LKAS_HUD']['CHECKSUM'], std)
       self.assertEqual(parser.vl['LKAS_HUD_A']['CHECKSUM'], ext)

can/tests/test_packer_parser.py

@@ -2,35 +2,13 @@
 import unittest
 import random
 
-import cereal.messaging as messaging
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
 from opendbc.can.tests import TEST_DBC
 
 MAX_BAD_COUNTER = 5
 
 
-# Python implementation so we don't have to depend on boardd
-def can_list_to_can_capnp(can_msgs, msgtype='can', logMonoTime=None):
-  dat = messaging.new_message(msgtype, len(can_msgs))
-
-  if logMonoTime is not None:
-    dat.logMonoTime = logMonoTime
-
-  for i, can_msg in enumerate(can_msgs):
-    if msgtype == 'sendcan':
-      cc = dat.sendcan[i]
-    else:
-      cc = dat.can[i]
-
-    cc.address = can_msg[0]
-    cc.busTime = can_msg[1]
-    cc.dat = bytes(can_msg[2])
-    cc.src = can_msg[3]
-
-  return dat.to_bytes()
-
-
 class TestCanParserPacker(unittest.TestCase):
   def test_packer(self):
     packer = CANPacker(TEST_DBC)
@@ -51,8 +29,7 @@ def test_packer_counter(self):
     # packer should increment the counter
     for i in range(1000):
       msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {})
-      dat = can_list_to_can_capnp([msg, ])
-      parser.update_strings([dat])
+      parser.update_strings([0, [msg]])
       self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], i % 256)
 
     # setting COUNTER should override
@@ -61,16 +38,14 @@ def test_packer_counter(self):
       msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {
         "COUNTER": cnt,
       })
-      dat = can_list_to_can_capnp([msg, ])
-      parser.update_strings([dat])
+      parser.update_strings([0, [msg]])
       self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], cnt)
 
     # then, should resume counting from the override value
     cnt = parser.vl["CAN_FD_MESSAGE"]["COUNTER"]
     for i in range(100):
       msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {})
-      dat = can_list_to_can_capnp([msg, ])
-      parser.update_strings([dat])
+      parser.update_strings([0, [msg]])
       self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], (cnt + i) % 256)
 
   def test_parser_can_valid(self):
@@ -83,16 +58,14 @@ def test_parser_can_valid(self):
 
     # not valid until the message is seen
     for _ in range(100):
-      dat = can_list_to_can_capnp([])
-      parser.update_strings([dat])
+      parser.update_strings([0, []])
       self.assertFalse(parser.can_valid)
 
     # valid once seen
     for i in range(1, 100):
       t = int(0.01 * i * 1e9)
       msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {})
-      dat = can_list_to_can_capnp([msg, ], logMonoTime=t)
-      parser.update_strings([dat])
+      parser.update_strings([t, [msg]])
       self.assertTrue(parser.can_valid)
 
   def test_parser_counter_can_valid(self):
@@ -107,17 +80,15 @@ def test_parser_counter_can_valid(self):
     parser = CANParser("honda_civic_touring_2016_can_generated", msgs, 0)
 
     msg = packer.make_can_msg("STEERING_CONTROL", 0, {"COUNTER": 0})
-    bts = can_list_to_can_capnp([msg])
 
     # bad static counter, invalid once it's seen MAX_BAD_COUNTER messages
     for idx in range(0x1000):
-      parser.update_strings([bts])
+      parser.update_strings([0, [msg]])
       self.assertEqual((idx + 1) < MAX_BAD_COUNTER, parser.can_valid)
 
     # one to recover
     msg = packer.make_can_msg("STEERING_CONTROL", 0, {"COUNTER": 1})
-    bts = can_list_to_can_capnp([msg])
-    parser.update_strings([bts])
+    parser.update_strings([0, [msg]])
     self.assertTrue(parser.can_valid)
 
   def test_parser_no_partial_update(self):
@@ -139,8 +110,7 @@ def rx_steering_msg(values, bad_checksum=False):
         msg[2] = bytearray(msg[2])
         msg[2][4] = (msg[2][4] & 0xF0) | ((msg[2][4] & 0x0F) + 1)
 
-      bts = can_list_to_can_capnp([msg])
-      parser.update_strings([bts])
+      parser.update_strings([0, [msg]])
 
     rx_steering_msg({"STEER_TORQUE": 100}, bad_checksum=False)
     self.assertEqual(parser.vl["STEERING_CONTROL"]["STEER_TORQUE"], 100)
@@ -183,8 +153,7 @@ def test_packer_parser(self):
         }
 
         msgs = [packer.make_can_msg(k, 0, v) for k, v in values.items()]
-        bts = can_list_to_can_capnp(msgs)
-        parser.update_strings([bts])
+        parser.update_strings([0, msgs])
 
         for k, v in values.items():
           for key, val in v.items():
@@ -204,9 +173,7 @@ def test_scale_offset(self):
     for brake in range(100):
       values = {"USER_BRAKE": brake}
       msgs = packer.make_can_msg("VSA_STATUS", 0, values)
-      bts = can_list_to_can_capnp([msgs])
-
-      parser.update_strings([bts])
+      parser.update_strings([0, [msgs]])
 
       self.assertAlmostEqual(parser.vl["VSA_STATUS"]["USER_BRAKE"], brake)
 
@@ -230,8 +197,7 @@ def test_subaru(self):
         }
 
         msgs = packer.make_can_msg("ES_LKAS", 0, values)
-        bts = can_list_to_can_capnp([msgs])
-        parser.update_strings([bts])
+        parser.update_strings([0, [msgs]])
 
         self.assertAlmostEqual(parser.vl["ES_LKAS"]["LKAS_Output"], steer)
         self.assertAlmostEqual(parser.vl["ES_LKAS"]["LKAS_Request"], active)
@@ -260,8 +226,7 @@ def send_msg(blank=False):
       else:
         msgs = [packer.make_can_msg("VSA_STATUS", 0, {}), ]
 
-      can = can_list_to_can_capnp(msgs, logMonoTime=t)
-      parser.update_strings([can, ])
+      parser.update_strings([t, msgs])
 
     # all good, no timeout
     for _ in range(1000):
@@ -299,8 +264,7 @@ def test_updated(self):
           can_msgs[frame].append(packer.make_can_msg("VSA_STATUS", 0, values))
           idx += 1
 
-      can_strings = [can_list_to_can_capnp(msgs) for msgs in can_msgs]
-      parser.update_strings(can_strings)
+      parser.update_strings([[0, can_msgs[0]], [0, can_msgs[1]]])
       vl_all = parser.vl_all["VSA_STATUS"]["USER_BRAKE"]
 
       self.assertEqual(vl_all, user_brake_vals)
@@ -334,7 +298,7 @@ def test_timestamp_nanos(self):
       for i in range(10):
         log_mono_time = int(0.01 * i * 1e+9)
         can_msg = packer.make_can_msg("VSA_STATUS", 0, {})
-        can_strings.append(can_list_to_can_capnp([can_msg], logMonoTime=log_mono_time))
+        can_strings.append((log_mono_time, [can_msg]))
       parser.update_strings(can_strings)
 
       ts_nanos = parser.ts_nanos["VSA_STATUS"].values()

can/tests/test_parser_performance.py

@@ -4,7 +4,6 @@
 
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
-from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp
 
 
 @unittest.skip("TODO: varies too much between machines")
@@ -17,8 +16,7 @@ def _benchmark(self, checks, thresholds, n):
     for i in range(50000):
       values = {"ACC_CONTROL": {"ACC_TYPE": 1, "ALLOW_LONG_PRESS": 3}}
       msgs = [packer.make_can_msg(k, 0, v) for k, v in values.items()]
-      bts = can_list_to_can_capnp(msgs, logMonoTime=int(0.01 * i * 1e9))
-      can_msgs.append(bts)
+      can_msgs.append([int(0.01 * i * 1e9), msgs])
 
     ets = []
     for _ in range(25):