Merge pull request #29 from andrewboutros/dev-abnash

Multi-RAD RADSim from dev-abnash

docs/rad-sim-developer.rst

@@ -6,6 +6,9 @@ RAD-Sim Testing Infrastructure
 
 Python Scripts Tests
 ^^^^^^^^^^^^^^^^^^^^^
+.. note::
+    This script does not currently work in multi-RAD RAD-Sim.
+
 To run python tests, ensure the current working directory is in the ``rad-sim`` folder and run the following steps:
 
 #. ``python -m unittest discover .``

docs/rad-sim-quick-start.rst

@@ -74,7 +74,6 @@ Building RAD-Sim
 ----------------
 
 You can configure RAD-Sim for your example design simulation using the following commands executed at the ``rad-sim`` root directory (the commands use the ``mlp`` example design which can be replaced by your own design under the ``rad-flow/rad-sim/example-designs`` directory):
-
 .. code-block:: bash
 
     $ cd <rad_flow_root_dir>/rad-sim

docs/rad-sim-rtl-code.rst

@@ -58,6 +58,17 @@ RAD-Sim has a pre-defined file structure for supporting RTL code. All RTL code m
 
 An example design that utilizes RTL modules can be found in the ``rad-sim/example-designs/rtl_add`` folder.
 
+.. note::
+   For designs containing multiple RADs,  RAD-Sim adds a portal module to each RAD for communication between devices.
+   Bitwidths for AXI-S signals carrying destination address (``tdest``) should match the ``DESTW`` set in 
+   ``sim/radsim_defines.hpp``, which is generated by running:
+      .. code-block:: bash
+      $ python config.py <design_name>
+
+.. note::
+   RAD-Sim adds a portal module for designs containing multiple RADs. The NoC, clock, and general configuration files
+   should be modified according to the code structure guide.
+
 RTL CMakeLists
 ---------------
 The RTL source folder additionally contains a CMakeLists script, and an optional port mapping file used for :ref:`automatic wrapper generation <auto_wrapper_gen>`.

docs/rad-sim-two-rad-dlrm-example.rst

@@ -0,0 +1,48 @@
+Two-RAD DLRM Example Design
+=================
+
+This guide explains how to use the two-RAD DLRM example design. RAD 1 is responsible for the DLRM up to and including the embedding table lookups.
+These are then transmitted to RAD 2 over the inter-RAD network, which then completes the remaining model stages.
+
+Building RAD-Sim
+----------------
+
+You can configure RAD-Sim for the two-RAD DLRM design simulation using the following commands executed at the ``rad-sim`` root directory.
+
+.. code-block:: bash
+
+    $ cd <rad_flow_root_dir>/rad-sim
+    $ python config.py dlrm_two_rad #dlrm_two_rad is name of design directory within example-designs parent directory
+
+Running RAD-Sim
+----------------
+
+You can then simulate this two-RAD DLRM example design following these steps:
+
+
+1. Generate a DLRM test case using the provided compiler:
+
+    .. code-block:: bash
+
+        $ cd <rad_flow_root_dir>/rad-sim/example-designs/dlrm_two_rad/compiler
+        $ python dlrm.py
+
+2. Run RAD-Sim simulation:
+
+    .. code-block:: bash
+
+        $ cd <rad_flow_root_dir>/rad-sim/build
+        $ make run
+        # Info: /OSCI/SystemC: Simulation stopped by user.
+        # Simulation Cycles from main.cpp = 20390
+        # [100%] Built target run
+        # dlrm_system.driver: Finished sending all inputs to embedding lookup module!
+        # dlrm_system.dut.feature_interaction_inst: Got all memory responses at cycle 6113!
+        # [==================================================] 100 %
+        # Got 2048 output(s)!
+        # Simulation PASSED! All outputs matching!
+        # Simulated 19958 cycle(s)
+
+        # Info: /OSCI/SystemC: Simulation stopped by user.
+        # Simulation Cycles from main.cpp = 19971
+        # [100%] Built target run

rad-sim/.gitignore

@@ -3,4 +3,8 @@
 radsim_knobs
 __pycache__
 
-test/*.xml
+test/*.xml
+*.cmake
+*.a
+CMakeFiles
+MakeFile

rad-sim/CMakeLists.txt

@@ -5,8 +5,11 @@ project(RADSim)
 set(CMAKE_BINARY_DIR "./build/")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 
-SET(DESIGN "dlrm" CACHE STRING "Design directory to be compiled. Must be under rad-flow/rad-sim/example-designs")
-message(STATUS "Compiling the ${DESIGN} design")
+#SET(DESIGN "dlrm" CACHE STRING "Design directory to be compiled. Must be under rad-flow/rad-sim/example-designs")
+FOREACH(DESIGN_NAME ${DESIGN_NAMES})
+    #MESSAGE("<<${DESIGN_NAME}>>")
+    message(STATUS "Compiling the ${DESIGN_NAME} design")
+ENDFOREACH()
 
 add_subdirectory(sim)
 add_subdirectory(example-designs)

rad-sim/config.yml

@@ -0,0 +1,63 @@
+config rad1:
+  dram:
+    num_controllers: 4
+    clk_periods: [3.32, 3.32, 2.0, 2.0]
+    queue_sizes: [64, 64, 64, 64]
+    config_files: ['DDR4_8Gb_x16_2400', 'DDR4_8Gb_x16_2400', 'HBM2_8Gb_x128', 'HBM2_8Gb_x128']
+
+  design:
+    name: 'dlrm_two_rad'
+    noc_placement: ['dlrm_two_rad.place']
+    clk_periods: [5.0, 2.0, 3.32, 1.5]
+
+config anotherconfig:
+  dram:
+    num_controllers: 4
+    clk_periods: [3.32, 3.32, 2.0, 2.0]
+    queue_sizes: [64, 64, 64, 64]
+    config_files: ['DDR4_8Gb_x16_2400', 'DDR4_8Gb_x16_2400', 'HBM2_8Gb_x128', 'HBM2_8Gb_x128']
+
+  design:
+    name: 'dlrm_two_rad'
+    noc_placement: ['dlrm_two_rad.place']
+    clk_periods: [5.0, 2.0, 3.32, 1.5]
+
+noc:
+  type: ['2d']
+  num_nocs: 1
+  clk_period: [1.0]
+  payload_width: [82]
+  topology: ['mesh']
+  dim_x: [10]
+  dim_y: [10] 
+  routing_func: ['dim_order']
+  vcs: [5]
+  vc_buffer_size: [16]
+  output_buffer_size: [8]
+  num_packet_types: [5]
+  router_uarch: ['iq']
+  vc_allocator: ['islip']
+  sw_allocator:  ['islip']
+  credit_delay: [1]
+  routing_delay: [1]
+  vc_alloc_delay: [1]
+  sw_alloc_delay: [1]
+
+noc_adapters:
+  clk_period: [1.25]
+  fifo_size: [16]
+  obuff_size: [2]
+  in_arbiter: ['fixed_rr']
+  out_arbiter: ['priority_rr']
+  vc_mapping: ['direct']
+
+cluster:  
+  sim_driver_period: 5.0
+  telemetry_log_verbosity: 2
+  telemetry_traces: ['Embedding LU', 'Mem0', 'Mem1', 'Mem2', 'Mem3', 'Feature Inter.', 'MVM first', 'MVM last']
+  num_rads: 2
+  cluster_configs: ['rad1', 'anotherconfig']
+  cluster_topology: 'all-to-all'
+  inter_rad_latency: 2100
+  inter_rad_bw: 102.4
+  inter_rad_fifo_num_slots: 1000

rad-sim/example-designs/CMakeLists.txt

@@ -1,4 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 find_package(SystemCLanguage CONFIG REQUIRED)
 
-add_subdirectory(${DESIGN})
+FOREACH(DESIGN_NAME ${DESIGN_NAMES})
+    MESSAGE("<<${DESIGN_NAME}>>")
+    add_subdirectory(${DESIGN_NAME})
+ENDFOREACH()

rad-sim/example-designs/add/.gitignore

@@ -0,0 +1,3 @@
+CMakeFiles/
+Makefile
+CMakeCache.txt

rad-sim/example-designs/add/CMakeLists.txt

@@ -31,5 +31,5 @@ set(hdrfiles
 
 add_compile_options(-Wall -Wextra -pedantic)
 
-add_library(design STATIC ${srcfiles} ${hdrfiles})
-target_link_libraries(design PUBLIC SystemC::systemc booksim noc)
+add_library(add STATIC ${srcfiles} ${hdrfiles})
+target_link_libraries(add PUBLIC SystemC::systemc booksim noc)

rad-sim/example-designs/add/add_driver.cpp

@@ -2,8 +2,14 @@
 
 #define NUM_ADDENDS 3
 
-add_driver::add_driver(const sc_module_name &name)
+add_driver::add_driver(const sc_module_name &name, RADSimDesignContext* radsim_design_)
     : sc_module(name) {
+
+  this->radsim_design = radsim_design_;
+
+  //for simulation cycle count
+  start_cycle = 0;
+  end_cycle = 0;
 
   // Random Seed
   srand (time(NULL));
@@ -31,7 +37,7 @@ void add_driver::source() {
   client_valid.write(false);
   wait();
   rst.write(false);
-  start_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
+  start_cycle = GetSimulationCycle(radsim_config.GetDoubleKnobShared("sim_driver_period"));
   start_time = std::chrono::steady_clock::now();
   wait();
 
@@ -52,24 +58,29 @@ void add_driver::source() {
 }
 
 void add_driver::sink() {
-  while (!response_valid.read()) {
+  while (!(response_valid.read())) {
     wait();
   }
   //std::cout << "Received " << response.read().to_uint64() << " sum from the adder!" << std::endl;
   //std::cout << "The actual sum is " << actual_sum << std::endl;
 
-  if (response.read() != actual_sum) {
+ if (response.read() != actual_sum) {
     std::cout << "FAILURE - Output is not matching!" << std::endl;
-    radsim_design.ReportDesignFailure();
+    radsim_design->ReportDesignFailure();
   } else {
     std::cout << "SUCCESS - Output is matching!" << std::endl;
   }
 
-  end_cycle = GetSimulationCycle(radsim_config.GetDoubleKnob("sim_driver_period"));
+  end_cycle = GetSimulationCycle(radsim_config.GetDoubleKnobShared("sim_driver_period"));
   end_time = std::chrono::steady_clock::now();
   std::cout << "Simulation Cycles = " << end_cycle - start_cycle << std::endl;
   std::cout << "Simulation Time = " << std::chrono::duration_cast<std::chrono::microseconds> (end_time - start_time).count() << " us" << std::endl;
   NoCTransactionTelemetry::DumpStatsToFile("stats.csv");
 
-  sc_stop();
+  end_cycle = GetSimulationCycle(radsim_config.GetDoubleKnobShared("sim_driver_period"));
+  std::cout << "Simulation Cycles for Just Adder Portion = " << end_cycle - start_cycle << std::endl;
+
+  this->radsim_design->set_rad_done();
+  return;
+
 }

rad-sim/example-designs/add/add_driver.hpp

@@ -10,10 +10,11 @@
 
 class add_driver : public sc_module {
 private:
+  int start_cycle, end_cycle;
   std::queue<int> numbers_to_send;
   int actual_sum;
-  int start_cycle, end_cycle;
   std::chrono::steady_clock::time_point start_time, end_time;
+  RADSimDesignContext* radsim_design;
 
 public:
   sc_in<bool> clk;
@@ -25,7 +26,7 @@ class add_driver : public sc_module {
   sc_in<sc_bv<DATAW>> response;
   sc_in<bool> response_valid;
 
-  add_driver(const sc_module_name &name);
+  add_driver(const sc_module_name &name, RADSimDesignContext* radsim_design_);
   ~add_driver();
 
   void source();

rad-sim/example-designs/add/add_system.cpp

@@ -1,10 +1,10 @@
 #include <add_system.hpp>
 
-add_system::add_system(const sc_module_name &name, sc_clock *driver_clk_sig)
+add_system::add_system(const sc_module_name &name, sc_clock *driver_clk_sig, RADSimDesignContext* radsim_design) 
     : sc_module(name) {
 
   // Instantiate driver
-  driver_inst = new add_driver("driver");
+  driver_inst = new add_driver("driver", radsim_design);
   driver_inst->clk(*driver_clk_sig);
   driver_inst->rst(rst_sig);
   driver_inst->client_tdata(client_tdata_sig);
@@ -15,14 +15,16 @@ add_system::add_system(const sc_module_name &name, sc_clock *driver_clk_sig)
   driver_inst->response_valid(response_valid_sig);
 
   // Instantiate design top-level
-  dut_inst = new add_top("dut");
+  dut_inst = new add_top("dut", radsim_design);
   dut_inst->rst(rst_sig);
   dut_inst->client_tdata(client_tdata_sig);
   dut_inst->client_tlast(client_tlast_sig);
   dut_inst->client_valid(client_valid_sig);
   dut_inst->client_ready(client_ready_sig);
   dut_inst->response(response_sig);
   dut_inst->response_valid(response_valid_sig);
+  //add add_top as dut instance for parent class RADSimDesignSystem
+  this->design_dut_inst = dut_inst;
 }
 
 add_system::~add_system() {

rad-sim/example-designs/add/add_system.hpp

@@ -4,8 +4,9 @@
 #include <add_top.hpp>
 #include <chrono>
 #include <vector>
+#include <design_system.hpp>
 
-class add_system : public sc_module {
+class add_system : public RADSimDesignSystem {
 private:
   sc_signal<sc_bv<DATAW>> client_tdata_sig;
   sc_signal<bool> client_tlast_sig;
@@ -21,6 +22,6 @@ class add_system : public sc_module {
   add_top *dut_inst;
 
   add_system(const sc_module_name &name,
-                           sc_clock *driver_clk_sig);
+                           sc_clock *driver_clk_sig, RADSimDesignContext* radsim_design);
   ~add_system();
 };

rad-sim/example-designs/add/add_top.cpp

@@ -1,15 +1,17 @@
 #include <add_top.hpp>
 
-add_top::add_top(const sc_module_name &name)
-    : sc_module(name) {
+add_top::add_top(const sc_module_name &name, RADSimDesignContext* radsim_design)
+    : RADSimDesignTop(radsim_design) {
+
+  this->radsim_design = radsim_design;
 
   std::string module_name_str;
   char module_name[25];
 
   module_name_str = "client_inst";
   std::strcpy(module_name, module_name_str.c_str());
 
-  client_inst = new client(module_name);
+  client_inst = new client(module_name, radsim_design);
   client_inst->rst(rst);
   client_inst->client_tdata(client_tdata);
   client_inst->client_tlast(client_tlast);
@@ -18,15 +20,15 @@ add_top::add_top(const sc_module_name &name)
 
   module_name_str = "adder_inst";
   std::strcpy(module_name, module_name_str.c_str());
-  adder_inst = new adder(module_name);
+  adder_inst = new adder(module_name, radsim_design);
   adder_inst->rst(rst);
   adder_inst->response(response);
   adder_inst->response_valid(response_valid);
 
-  radsim_design.BuildDesignContext("add.place",
-                                   "add.clks");
-  radsim_design.CreateSystemNoCs(rst);
-  radsim_design.ConnectModulesToNoC();
+  this->connectPortalReset(&rst);
+  radsim_design->BuildDesignContext("add.place", "add.clks");
+  radsim_design->CreateSystemNoCs(rst);
+  radsim_design->ConnectModulesToNoC();
 }
 
 add_top::~add_top() {