From 91aaef67a6296a42ac1489ca6a3a23e063c8010c Mon Sep 17 00:00:00 2001 From: Mike Busuttil Date: Thu, 9 Jan 2025 22:38:41 -0500 Subject: [PATCH 1/4] "install" ctre instead of re2 --- .gitignore | 1 + test.sh | 12 +----------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 63e31bb543..048e499939 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ opendbc/can/packer_pyx.cpp opendbc/can/parser_pyx.cpp opendbc/can/packer_pyx.html opendbc/can/parser_pyx.html +opendbc/can/ctre.hpp opendbc/dbc/*_generated.dbc \ No newline at end of file diff --git a/test.sh b/test.sh index 284da0653b..feb21e1a79 100755 --- a/test.sh +++ b/test.sh @@ -10,17 +10,7 @@ fi uv sync --all-extras source .venv/bin/activate - -mkdir -p .tmp -echo ' -#include -RE2 x("");int main(void) {return 0;} -' > .tmp/re2.c -g++ -o .tmp/re2.o .tmp/re2.c -lre2 &>/dev/null || { - echo "'re2' is not installed. Installing 're2'..." - [[ $OSTYPE = "linux-gnu" ]] && sudo apt-get install -y --no-install-recommends libre2-dev || brew install re2 -} -rm -rf .tmp +[ ! -f opendbc/can/ctre.hpp ] && wget -O opendbc/can/ctre.hpp https://github.com/hanickadot/compile-time-regular-expressions/raw/refs/heads/main/single-header/ctre.hpp # *** build *** uv run scons -j$(nproc 2>/dev/null || sysctl -n hw.logicalcpu) From 78c0f29543f802499c3133b3f4df80e53845cf41 Mon Sep 17 00:00:00 2001 From: Mike Busuttil Date: Fri, 10 Jan 2025 23:12:11 -0500 Subject: [PATCH 2/4] replace all but val_split --- .gitignore | 2 +- opendbc/can/dbc.cc | 49 +++++++++++++++++++++++----------------------- test.sh | 2 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 048e499939..91505bcbc3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ *.egg-info/ *.html uv.lock +ctre.hpp opendbc/can/*.so opendbc/can/*.a @@ -21,5 +22,4 @@ opendbc/can/packer_pyx.cpp opendbc/can/parser_pyx.cpp opendbc/can/packer_pyx.html opendbc/can/parser_pyx.html -opendbc/can/ctre.hpp opendbc/dbc/*_generated.dbc \ No newline at end of file diff --git a/opendbc/can/dbc.cc b/opendbc/can/dbc.cc index 1934c5b34e..94715c52b6 100644 --- a/opendbc/can/dbc.cc +++ b/opendbc/can/dbc.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,14 +11,16 @@ #include #include #include - +#include #include "opendbc/can/common.h" #include "opendbc/can/common_dbc.h" -RE2 bo_regexp(R"(^BO_ (\w+) (\w+) *: (\w+) (\w+))"); -RE2 sg_regexp(R"(^SG_ (\w+) : (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*))"); -RE2 sgm_regexp(R"(^SG_ (\w+) (\w+) *: (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*))"); -RE2 val_regexp(R"(VAL_ (\w+) (\w+) (.*))"); +constexpr auto bo_pattern = ctll::fixed_string{R"(^BO_ (\w+) (\w+) *: (\w+) \w+)"}; +constexpr auto sg_pattern = ctll::fixed_string{R"(^SG_ (\w+) .*: (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[[0-9.+\-eE]+\|[0-9.+\-eE]+\] \".*\" .*)"}; +constexpr auto val_pattern = ctll::fixed_string{R"(VAL_ (\w+) (\w+) (.*))"}; +constexpr auto bo_match = ctre::match; +constexpr auto sg_match = ctre::match; +constexpr auto val_match = ctre::match; RE2 val_split_regexp(R"((([0-9]) \"(.+?)\"))"); #define DBC_ASSERT(condition, message) \ @@ -128,13 +131,13 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch line_num += 1; if (startswith(line, "BO_ ")) { // new group - bool ret = RE2::FullMatch(line, bo_regexp, &match1, &match2, &match3); + auto [ret, match01, match02, match03] = bo_match(line); DBC_ASSERT(ret, "bad BO: " << line); Msg& msg = dbc->msgs.emplace_back(); - address = msg.address = std::stoul(match1); // could be hex - msg.name = match2; - msg.size = std::stoul(match3); + address = msg.address = std::stoul(match01.str()); // could be hex + msg.name = match02.str(); + msg.size = std::stoul(match03.str()); // check for duplicates DBC_ASSERT(address_set.find(address) == address_set.end(), "Duplicate message address: " << address << " (" << msg.name << ")"); @@ -146,18 +149,16 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch } } else if (startswith(line, "SG_ ")) { // new signal - if (!RE2::FullMatch(line, sg_regexp, &match1, &match2, &match3, &match4, &match5, &match6, &match7)) { - bool ret = RE2::FullMatch(line, sgm_regexp, &match1, &ignore, &match2, &match3, &match4, &match5, &match6, &match7); - DBC_ASSERT(ret, "bad SG: " << line); - } + auto [ret, match01, match02, match03, match04, match05, match06, match07] = sg_match(line); + DBC_ASSERT(ret, "bad SG: " << line); Signal& sig = signals[address].emplace_back(); - sig.name = match1; - sig.start_bit = std::stoi(match2); - sig.size = std::stoi(match3); - sig.is_little_endian = std::stoi(match4) == 1; - sig.is_signed = match5 == "-"; - sig.factor = std::stod(match6); - sig.offset = std::stod(match7); + sig.name = match01.str(); + sig.start_bit = std::stoi(match02.str()); + sig.size = std::stoi(match03.str()); + sig.is_little_endian = std::stoi(match04.str()) == 1; + sig.is_signed = match05.str() == "-"; + sig.factor = std::stod(match06.str()); + sig.offset = std::stod(match07.str()); set_signal_type(sig, checksum, dbc_name, line_num); if (sig.is_little_endian) { sig.lsb = sig.start_bit; @@ -174,14 +175,14 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch signal_name_sets[address].insert(sig.name); } else if (startswith(line, "VAL_ ")) { // new signal value/definition - bool ret = RE2::FullMatch(line, val_regexp, &match1, &match2, &match3); + auto [ret, match01, match02, match03] = val_match(line); DBC_ASSERT(ret, "bad VAL: " << line); auto& val = dbc->vals.emplace_back(); - val.address = std::stoul(match1); // could be hex - val.name = match2; + val.address = std::stoul(match01.str()); // could be hex + val.name = match02.str(); - auto defvals = match3; + auto defvals = match03.str(); // convert strings to UPPER_CASE_WITH_UNDERSCORES std::vector words; std::string full_match, number, word; diff --git a/test.sh b/test.sh index feb21e1a79..95019130c5 100755 --- a/test.sh +++ b/test.sh @@ -10,7 +10,7 @@ fi uv sync --all-extras source .venv/bin/activate -[ ! -f opendbc/can/ctre.hpp ] && wget -O opendbc/can/ctre.hpp https://github.com/hanickadot/compile-time-regular-expressions/raw/refs/heads/main/single-header/ctre.hpp +[ ! -f ctre.hpp ] && wget https://github.com/hanickadot/compile-time-regular-expressions/raw/refs/heads/main/single-header/ctre.hpp # *** build *** uv run scons -j$(nproc 2>/dev/null || sysctl -n hw.logicalcpu) From bcfa3447386ec11ddb782f169e4d6a65a561974a Mon Sep 17 00:00:00 2001 From: Mike Busuttil Date: Sat, 11 Jan 2025 18:15:25 -0500 Subject: [PATCH 3/4] tokenize def_vals --- opendbc/can/dbc.cc | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/opendbc/can/dbc.cc b/opendbc/can/dbc.cc index c9d5df3126..8d8c854a1f 100644 --- a/opendbc/can/dbc.cc +++ b/opendbc/can/dbc.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -18,10 +17,11 @@ constexpr auto bo_pattern = ctll::fixed_string{R"(^BO_ (\w+) (\w+) *: (\w+) \w+)"}; constexpr auto sg_pattern = ctll::fixed_string{R"(^SG_ (\w+) .*: (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[[0-9.+\-eE]+\|[0-9.+\-eE]+\] \".*\" .*)"}; constexpr auto val_pattern = ctll::fixed_string{R"(VAL_ (\w+) (\w+) (.*))"}; +constexpr auto val_split_pattern = ctll::fixed_string{R"( ?(?:(\d+) \"([^"]++)\"))"}; constexpr auto bo_match = ctre::match; constexpr auto sg_match = ctre::match; constexpr auto val_match = ctre::match; -RE2 val_split_regexp(R"((([0-9]) \"(.+?)\"))"); +constexpr auto val_tokenize = ctre::tokenize; #define DBC_ASSERT(condition, message) \ do { \ @@ -124,19 +124,18 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch std::string line; int line_num = 0; - std::string match1, match2, match3, match4, match5, match6, match7; while (std::getline(stream, line)) { line = trim(line); line_num += 1; if (startswith(line, "BO_ ")) { // new group - auto [ret, match01, match02, match03] = bo_match(line); + auto [ret, match1, match2, match3] = bo_match(line); DBC_ASSERT(ret, "bad BO: " << line); Msg& msg = dbc->msgs.emplace_back(); - address = msg.address = std::stoul(match01.str()); // could be hex - msg.name = match02.str(); - msg.size = std::stoul(match03.str()); + address = msg.address = std::stoul(match1.str()); // could be hex + msg.name = match2.str(); + msg.size = std::stoul(match3.str()); // check for duplicates DBC_ASSERT(address_set.find(address) == address_set.end(), "Duplicate message address: " << address << " (" << msg.name << ")"); @@ -148,16 +147,16 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch } } else if (startswith(line, "SG_ ")) { // new signal - auto [ret, match01, match02, match03, match04, match05, match06, match07] = sg_match(line); + auto [ret, match1, match2, match3, match4, match5, match6, match7] = sg_match(line); DBC_ASSERT(ret, "bad SG: " << line); Signal& sig = signals[address].emplace_back(); - sig.name = match01.str(); - sig.start_bit = std::stoi(match02.str()); - sig.size = std::stoi(match03.str()); - sig.is_little_endian = std::stoi(match04.str()) == 1; - sig.is_signed = match05.str() == "-"; - sig.factor = std::stod(match06.str()); - sig.offset = std::stod(match07.str()); + sig.name = match1.str(); + sig.start_bit = std::stoi(match2.str()); + sig.size = std::stoi(match3.str()); + sig.is_little_endian = std::stoi(match4.str()) == 1; + sig.is_signed = match5.str() == "-"; + sig.factor = std::stod(match6.str()); + sig.offset = std::stod(match7.str()); set_signal_type(sig, checksum, dbc_name, line_num); if (sig.is_little_endian) { sig.lsb = sig.start_bit; @@ -174,23 +173,22 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch signal_name_sets[address].insert(sig.name); } else if (startswith(line, "VAL_ ")) { // new signal value/definition - auto [ret, match01, match02, match03] = val_match(line); + auto [ret, match1, match2, match3] = val_match(line); DBC_ASSERT(ret, "bad VAL: " << line); auto& val = dbc->vals.emplace_back(); - val.address = std::stoul(match01.str()); // could be hex - val.name = match02.str(); + val.address = std::stoul(match1.str()); // could be hex + val.name = match2.str(); - auto defvals = match03.str(); + auto defvals = match3.str(); // convert strings to UPPER_CASE_WITH_UNDERSCORES std::vector words; - std::string full_match, number, word; - while (RE2::PartialMatch(defvals, val_split_regexp, &full_match, &number, &word)) { + for (auto match : val_tokenize(defvals)) { + std::string word = match.get<2>().str(); word = trim(word); std::transform(word.begin(), word.end(), word.begin(), ::toupper); std::replace(word.begin(), word.end(), ' ', '_'); - words.push_back(number + " " + word); - defvals = defvals.substr(full_match.length(), defvals.length() - full_match.length()); + words.push_back(match.get<1>().str() + " " + word); } // join string std::stringstream s; From c1edfdcca7e56dc50112ce988928d7d027a157cc Mon Sep 17 00:00:00 2001 From: Mike Busuttil Date: Sun, 12 Jan 2025 21:52:58 -0500 Subject: [PATCH 4/4] remove iostream --- opendbc/can/dbc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendbc/can/dbc.cc b/opendbc/can/dbc.cc index 8d8c854a1f..dc6b0d4cf7 100644 --- a/opendbc/can/dbc.cc +++ b/opendbc/can/dbc.cc @@ -10,7 +10,7 @@ #include #include #include -#include + #include "opendbc/can/common.h" #include "opendbc/can/common_dbc.h"