Merge pull request #21 from andrewboutros/dev-gtrieu

RTL Co-Simulation

.github/scripts/setup_verilator.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+sudo apt-get update && apt-get upgrade -y
+sudo apt install git help2man perl python3 make autoconf g++ flex bison ccache
+sudo apt install libgoogle-perftools-dev numactl perl-doc
+sudo apt install libfl2
+sudo apt install libfl-dev
+sudo apt install zlibc zlib1g zlib1g-dev
+
+git clone https://github.com/verilator/verilator
+unset VERILATOR_ROOT  # For bash
+cd verilator
+autoconf         # Create ./configure script
+./configure      # Configure and create Makefile
+make -j `nproc`  # Build Verilator itself (if error, try just 'make')
+sudo make install

.github/workflows/rad_sim_ci.yml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-script: [mlp_test.sh, dlrm_test.sh, npu_test.sh]
+        test-script: [mlp_test.sh, dlrm_test.sh, npu_test.sh, mlp_int8_test.sh]
         test: [""]
         exclude:
           - test-script: npu_test.sh # Don't run all the NPU tests in the same runner
@@ -76,6 +76,10 @@ jobs:
     - name: Setup SystemC
       run: ./.github/scripts/setup_system_c.sh -v ${{ env.SYSTEMC_VERSION }}
 
+    - name: Setup Verilator IFF test requires it
+      if: matrix.test-script == 'mlp_int8_test.sh'
+      run: ./.github/scripts/setup_verilator.sh
+
     - name: Setup Mambaforge (Conda)
       uses: conda-incubator/setup-miniconda@v2
       with:

docs/rad-sim-rtl-code.rst

@@ -55,11 +55,21 @@ RAD-Sim has a pre-defined file structure for supporting RTL code. All RTL code m
     ├── CMakeLists.txt
     └── config.yml
 
+An example design that utilizes RTL modules can be found in the ``rad-sim/example-designs/rtl_add`` folder.
+
+RTL CMakeLists
+---------------
 The RTL source folder additionally contains a CMakeLists script, and an optional port mapping file used for :ref:`automatic wrapper generation <auto_wrapper_gen>`.
 The CMakeLists script imports the required libraries and verilates the RTL designs to SystemC modules.
 These objects are linked in the design CMakeLists script.
 
-An example design that utilizes RTL modules can be found in the ``rad-sim/example-designs/rtl_add`` folder.
+Under ``rtl_top_modules``, a list of all top-level RTL modules should be placed. Each top-level module will be verilated to become a C++ SystemC module.
+All modules instantiated by a top-level module are merged into the SystemC module of the top-level design automatically.
+
+Verilator does not recognize modules with a file name other than the module name with a .v/.sv extension automatically.
+It is therefore recommended that a list of non top-level modules be added under ``rtl_libraries``.
+
+A sample CMakeLists file can be found in the ``rad-sim/example-designs/rtl_add/modules/rtl`` folder.
 
 Wrapper Files
 -------------
@@ -81,7 +91,7 @@ Automatic wrapper generation follows the workflow:
 
 #. Run ``generate_port_mappings.py`` with the design path and RTL design files as arguments.
 
-   * ex. ``python generate_port_mappings.py adder.v client.v``
+   * ex. ``python generate_port_mappings.py example-designs/rtl_add adder.v client.v``
 
 #. Check the console for inference warnings.
 
@@ -90,15 +100,15 @@ Automatic wrapper generation follows the workflow:
 
 #. Run ``generate_wrapper.py`` with the design path and module names as arguments.
 
-   * ex. ``python generate_wrapper.py adder client``
+   * ex. ``python generate_wrapper.py example-designs/rtl_add adder client``
    * Note: these are modules connected to the NoC.
 
 These scripts produce basic source and header wrapper files for the specified RTL modules.
 Advanced users may edit these files to add additional functionality.
 
 AXI-S Formatting Requirement
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Ports in RTL modules using AXI-S must be specified in the format ``axis_{name}_interface_{signal}`` to be recognized by the port mappings script.
+Ports in RTL modules using AXI-S must be specified in the format ``axis_{name}_{signal}`` to be recognized by the port mappings script.
 
 Port Map File Format
 ^^^^^^^^^^^^^^^^^^^^^
@@ -107,35 +117,35 @@ The port map file is a blank-space delimited file used to specify the connection
 * All inputs, outputs, and AXI-S ports must be under a ``module``.
 * Modules are defined by ``module {name}``.
 * Input and Output ports are defined by ``{input/output} {port_width} {rtl_port} {radsim_port}``.
-* AXI-S ports are defined by ``axis {master/slave} {rtl_port} {radsim_port}``.
+* AXI-S ports are defined by ``axis {master/slave} {rtl_port} {axis_interface} {axis_port}``.
 
 An example port map file from the ``rtl_add`` example is shown below:
 
 .. code-block::
 
-    module adder
-    input 1 clk clk
-    input 1 rst rst
-    axis slave axis_adder_interface_tvalid axis_adder_interface.tvalid
-    axis slave axis_adder_interface_tlast axis_adder_interface.tlast
-    axis slave axis_adder_interface_tdata axis_adder_interface.tdata
-    axis slave axis_adder_interface_tready axis_adder_interface.tready
-    output 128 response response
-    output 1 response_valid response_valid
-
-    module client
-    input 1 clk clk
-    input 1 rst rst
-    input 128 client_tdata client_tdata
-    input 1 client_tlast client_tlast
-    input 1 client_valid client_valid
-    axis master axis_client_interface_tready axis_client_interface.tready
-    output 1 client_ready client_ready
-    axis master axis_client_interface_tvalid axis_client_interface.tvalid
-    axis master axis_client_interface_tlast axis_client_interface.tlast
-    axis master axis_client_interface_tdest axis_client_interface.tdest
-    axis master axis_client_interface_tid axis_client_interface.tid
-    axis master axis_client_interface_tstrb axis_client_interface.tstrb
-    axis master axis_client_interface_tkeep axis_client_interface.tkeep
-    axis master axis_client_interface_tuser axis_client_interface.tuser
-    axis master axis_client_interface_tdata axis_client_interface.tdata
+   module adder
+   input 1 clk clk
+   input 1 rst rst
+   axis slave axis_adder_tvalid axis_adder tvalid
+   axis slave axis_adder_tlast axis_adder tlast
+   axis slave axis_adder_tdata axis_adder tdata
+   axis slave axis_adder_tready axis_adder tready
+   output 128 response response
+   output 1 response_valid response_valid
+
+   module client
+   input 1 clk clk
+   input 1 rst rst
+   input 128 client_tdata client_tdata
+   input 1 client_tlast client_tlast
+   input 1 client_valid client_valid
+   axis master axis_client_tready axis_client tready
+   output 1 client_ready client_ready
+   axis master axis_client_tvalid axis_client tvalid
+   axis master axis_client_tlast axis_client tlast
+   axis master axis_client_tdest axis_client tdest
+   axis master axis_client_tid axis_client tid
+   axis master axis_client_tstrb axis_client tstrb
+   axis master axis_client_tkeep axis_client tkeep
+   axis master axis_client_tuser axis_client tuser
+   axis master axis_client_tdata axis_client tdata

rad-sim/example-designs/add/add_driver.cpp

@@ -31,6 +31,8 @@ void add_driver::source() {
   client_valid.write(false);
   wait();
   rst.write(false);
+  start_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
+  start_time = std::chrono::steady_clock::now();
   wait();
 
   while (!numbers_to_send.empty()) {
@@ -45,19 +47,25 @@ void add_driver::source() {
     }
   }
   client_valid.write(false);
-  std::cout << "Finished sending all numbers to client module!" << std::endl;
+  //std::cout << "Finished sending all numbers to client module!" << std::endl;
   wait();
 }
 
 void add_driver::sink() {
   while (!response_valid.read()) {
     wait();
   }
-  std::cout << "Received " << response.read().to_uint64() << " sum from the adder!" << std::endl;
-  std::cout << "The actual sum is " << actual_sum << std::endl;
+  //std::cout << "Received " << response.read().to_uint64() << " sum from the adder!" << std::endl;
+  //std::cout << "The actual sum is " << actual_sum << std::endl;
 
   if (response.read() != actual_sum) std::cout << "FAILURE - Output is not matching!" << std::endl;
   else std::cout << "SUCCESS - Output is matching!" << std::endl;
 
+  end_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
+  end_time = std::chrono::steady_clock::now();
+  std::cout << "Simulation Cycles = " << end_cycle - start_cycle << std::endl;
+  std::cout << "Simulation Time = " << std::chrono::duration_cast<std::chrono::microseconds> (end_time - start_time).count() << " us" << std::endl;
+  NoCTransactionTelemetry::DumpStatsToFile("stats.csv");
+
   sc_stop();
 }

rad-sim/example-designs/add/add_driver.hpp

@@ -12,6 +12,8 @@ class add_driver : public sc_module {
 private:
   std::queue<int> numbers_to_send;
   int actual_sum;
+  int start_cycle, end_cycle;
+  std::chrono::steady_clock::time_point start_time, end_time;
 
 public:
   sc_in<bool> clk;

rad-sim/example-designs/add/modules/adder.cpp

@@ -40,10 +40,10 @@ void adder::Tick() {
       uint64_t current_sum = adder_rolling_sum.to_uint64();
       adder_rolling_sum = current_sum + axis_adder_interface.tdata.read().to_uint64();
       t_finished.write(axis_adder_interface.tlast.read());
-      std::cout << module_name << ": Got Transaction (user = "
-                << axis_adder_interface.tuser.read().to_uint64() << ") (addend = "
-                << axis_adder_interface.tdata.read().to_uint64() << ")!"
-                << std::endl;
+      //std::cout << module_name << ": Got Transaction (user = "
+      //          << axis_adder_interface.tuser.read().to_uint64() << ") (addend = "
+      //          << axis_adder_interface.tdata.read().to_uint64() << ")!"
+      //          << std::endl;
     }
 
     // Print Sum and Exit

rad-sim/example-designs/add/modules/client.cpp

@@ -21,7 +21,7 @@ client::~client() {}
 
 void client::Assign() {
   if (rst) {
-    client_ready.write(true); // ready to accept requests from driver testbench
+    client_ready.write(false);
   } else {
     // Ready to accept new addend from driver testbench as long as the addend
     // FIFO is not full
@@ -45,8 +45,8 @@ void client::Tick() {
     // Interface with testbench driver
     if (client_ready.read() && client_valid.read()) {
       client_tdata_fifo.push(client_tdata);
-      testbench_tlast = client_tlast.read();
-      std::cout << module_name << ": Pushed request to FIFO" << std::endl;
+      r_client_tlast = client_tlast.read();
+      //std::cout << module_name << ": Pushed request to FIFO" << std::endl;
     }
     client_fifo_full.write(client_tdata_fifo.size() >= client_fifo_depth);
 
@@ -62,7 +62,7 @@ void client::Tick() {
       axis_client_interface.tstrb.write(0);
       axis_client_interface.tkeep.write(0);
       axis_client_interface.tuser.write(src_addr);
-      axis_client_interface.tlast.write(testbench_tlast && (client_tdata_fifo.size() == 1));
+      axis_client_interface.tlast.write(r_client_tlast && (client_tdata_fifo.size() == 1));
       axis_client_interface.tdata.write(tdata);
 
       axis_client_interface.tvalid.write(true);
@@ -73,7 +73,7 @@ void client::Tick() {
     if (axis_client_interface.tvalid.read() &&
         axis_client_interface.tready.read()) {
       client_tdata_fifo.pop();
-      std::cout << module_name << ": Sent Transaction!" << std::endl;
+      //std::cout << module_name << ": Sent Transaction!" << std::endl;
     }
     wait();
   }

rad-sim/example-designs/add/modules/client.hpp

@@ -16,7 +16,7 @@ class client : public RADSimModule {
   std::queue<sc_bv<DATAW>> client_tdata_fifo; // FIFO to store numbers
   unsigned int client_fifo_depth;    // MAXIMUM number of addends to store in FIFO
   sc_signal<bool> client_fifo_full;  // Signal flagging addend FIFO is full
-  bool testbench_tlast;
+  bool r_client_tlast;
 
 public:
   sc_in<bool> rst;

rad-sim/example-designs/mlp/mlp_driver.cpp

@@ -82,7 +82,7 @@ void mlp_driver::source() {
     dispatcher_fifo_wen[dispatcher_id].write(false);
   wait();
   rst.write(false);
-  start_cycle = GetSimulationCycle(1.0);
+  start_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
   wait();
 
   std::vector<unsigned int> written_inputs(num_mvms[0], 0);
@@ -129,7 +129,7 @@ void mlp_driver::sink() {
   if (mistake) std::cout << "FAILURE - Some outputs NOT matching!" << std::endl;
   else std::cout << "SUCCESS - All outputs are matching!" << std::endl;
 
-  end_cycle = GetSimulationCycle(1.0);
+  end_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
   std::cout << "Simulation Cycles = " << end_cycle - start_cycle << std::endl;
   NoCTransactionTelemetry::DumpStatsToFile("stats.csv");
   NoCFlitTelemetry::DumpNoCFlitTracesToFile("flit_traces.csv");

rad-sim/example-designs/mlp_int8/CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.16)
 find_package(SystemCLanguage CONFIG REQUIRED)
 
+add_subdirectory(modules/rtl)
+
 include_directories(
   ./
   modules
@@ -11,6 +13,8 @@ include_directories(
   ../../sim/noc/booksim/routers
 )
 
+include_directories(SYSTEM "${VERILATOR_ROOT}/include")
+
 set(srcfiles
   modules/fifo.cpp
   modules/register_file.cpp
@@ -20,7 +24,8 @@ set(srcfiles
   modules/collector.cpp
   modules/datapath.cpp
   modules/pipeline.cpp
-  modules/mvm.cpp
+  modules/sysc_mvm.cpp
+  modules/rtl_mvm.cpp
   modules/weight_loader.cpp
   modules/inst_loader.cpp
   mlp_top.cpp
@@ -37,7 +42,8 @@ set(hdrfiles
   modules/collector.hpp
   modules/datapath.hpp
   modules/pipeline.hpp
-  modules/mvm.hpp
+  modules/sysc_mvm.hpp
+  modules/rtl_mvm.hpp
   modules/weight_loader.hpp
   modules/inst_loader.hpp
   modules/config.hpp
@@ -49,4 +55,4 @@ set(hdrfiles
 add_compile_options(-Wall -Wextra -pedantic)
 
 add_library(design STATIC ${srcfiles} ${hdrfiles})
-target_link_libraries(design PUBLIC SystemC::systemc booksim noc)
+target_link_libraries(design PUBLIC SystemC::systemc booksim noc rtl_designs)

rad-sim/example-designs/mlp_int8/compiler/gen_testcase.py

@@ -240,5 +240,5 @@
 config_file = open('./layer_mvm_config', 'w')
 config_file.write(str(num_layers) + ' ')
 for mvm_count in num_mvms:
-  config_file.write(str(mvm_count) + ' ')
+  config_file.write(str(mvm_count) + ',0 ') # by default, initialize all MVM instances to be all native SystemC modules
 config_file.close()

rad-sim/example-designs/mlp_int8/mlp.place

@@ -1,10 +1,10 @@
-layer0_mvm0 0 5 axis
-layer0_mvm1 0 13 axis
-layer1_mvm0 0 15 axis
-layer1_mvm1 0 7 axis
-layer2_mvm0 0 2 axis
-input_dispatcher0 0 14 axis
-input_dispatcher1 0 8 axis
-output_collector 0 9 axis
-weight_loader 0 10 axis
+layer0_mvm0 0 12 axis
+layer0_mvm1 0 2 axis
+layer1_mvm0 0 10 axis
+layer1_mvm1 0 14 axis
+layer2_mvm0 0 8 axis
+input_dispatcher0 0 0 axis
+input_dispatcher1 0 13 axis
+output_collector 0 7 axis
+weight_loader 0 6 axis
 inst_loader 0 4 axis