Merge pull request #18 from andrewboutros/dev-gtrieu

CI Improvements, Minor documentation changes

.github/scripts/run_test.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+TEST_ERROR=0
+if [ -z $TEST ]; then
+    ./rad-sim/test/$TEST_SCRIPT || TEST_ERROR=$?
+else
+    ./rad-sim/test/$TEST_SCRIPT -t $TEST || TEST_ERROR=$?
+fi
+sed -i "s/\x1B\[[0-9;]\{1,\}[A-Za-z]/**/g" $GITHUB_STEP_SUMMARY
+exit "$TEST_ERROR"

.github/workflows/rad_sim_ci.yml

@@ -8,6 +8,7 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -38,26 +39,42 @@ jobs:
         paths: "rad-sim/test/*report.xml"
         show: "all"
 
-  build:
+  design-tests:
     runs-on: ubuntu-latest
     defaults:
       run:
-        shell: bash -el {0}
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        test-script: [mlp_test.sh, dlrm_test.sh, npu_test.sh]
+        test: [""]
+        exclude:
+          - test-script: npu_test.sh # Don't run all the NPU tests in the same runner
+            test: ""
+        include:
+          - test-script: npu_test.sh
+            test: std_gemv
+          - test-script: npu_test.sh
+            test: std_rnn
+          - test-script: npu_test.sh
+            test: std_gru
+          - test-script: npu_test.sh
+            test: std_lstm
+          - test-script: npu_test.sh
+            test: std_mlp
     steps:
     - uses: actions/checkout@v3
 
-    - name: Set time
-      run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
-
     - name: Cache SystemC build files
       id: cache-systemc
       uses: actions/cache@v3
       with:
-        path: /home/runner/work/rad-flow/rad-flow/systemc-${{ env.SYSTEMC_VERSION }}/build
+        path: ${{ github.workspace }}/systemc-${{ env.SYSTEMC_VERSION }}/build
         key: ubuntu-64-systemc-${{ env.SYSTEMC_VERSION }}
 
     - name: Setup SystemC
-      run: ./scripts/setup_system_c.sh -v ${{ env.SYSTEMC_VERSION }}
+      run: ./.github/scripts/setup_system_c.sh -v ${{ env.SYSTEMC_VERSION }}
 
     - name: Setup Mambaforge (Conda)
       uses: conda-incubator/setup-miniconda@v2
@@ -78,17 +95,8 @@ jobs:
       if: steps.cache.outputs.cache-hit != 'true'
       run: mamba env update -n radflow -f rad-flow-env.yml
 
-    - name: Build RAD-Sim
-      run: |
-        cd rad-sim
-        python config.py mlp
-    
-    - name: Generate Test Cases
-      run: |
-        cd rad-sim/example-designs/mlp/compiler
-        python gen_testcase.py 4 512 512 512 256 128 4 3 2 2
-
-    - name: Run Tests
-      run: |
-        cd rad-sim/build
-        make run
+    - name: Run Test
+      env:
+        TEST_SCRIPT: ${{ matrix.test-script }}
+        TEST: ${{ matrix.test }}
+      run: ./.github/scripts/run_test.sh

docs/index.rst

@@ -47,7 +47,7 @@ The following paper may be used as a general citation for RAD-Sim:
    rad-sim-code-structure
    rad-sim-rtl-code
    rad-sim-adder-design
-   rad-sim-testing
+   rad-sim-developer
 
 .. toctree::
    :caption: RAD-Gen Documentation

docs/rad-sim-adder-design.rst

@@ -247,7 +247,16 @@ The example config provided in :doc:`rad-sim-code-structure` is used.
 
 Running the Example
 -------------------
-Running the example with ``NUM_ADDENDS = 3``, the following output is produced.
+Running the example with ``NUM_ADDENDS = 3``, and the commands:
+
+.. code-block:: bash
+
+    $ cd <rad_flow_root_dir>/rad-sim
+    $ python config.py add
+    $ cd build
+    $ make run
+
+The following output is produced.
 
 .. code-block:: text
 

docs/rad-sim-developer.rst

@@ -0,0 +1,48 @@
+RAD-Sim Developers
+===================
+
+RAD-Sim Testing Infrastructure
+-------------------------------
+
+Python Scripts Tests
+^^^^^^^^^^^^^^^^^^^^^
+To run python tests, ensure the current working directory is in the ``rad-sim`` folder and run the following steps:
+
+#. ``python -m unittest discover .``
+
+If you would like a test report XML to be generated, run the following steps:
+
+#. ``pip install -r test/requirements.txt``
+#. ``python -m xmlrunner discover . --output-file test/python-test-report.xml``
+
+MLP Tests
+^^^^^^^^^^
+To run the MLP test, ensure the current working directory is in the ``rad-sim`` folder and run the following steps:
+
+#. ``conda activate radflow``
+#. ``test/mlp_test.sh``
+
+DLRM Tests
+^^^^^^^^^^
+To run the DLRM test, ensure the current working directory is in the ``rad-sim`` folder and run the following steps:
+
+#. ``conda activate radflow``
+#. ``test/dlrm_test.sh``
+
+NPU Tests
+^^^^^^^^^^
+To run the NPU test, ensure the current working directory is in the ``rad-sim`` folder and run the following steps:
+
+#. ``conda activate radflow``
+#. ``test/npu_test.sh``
+
+If you would like to run specific tests (ex. std_gemv, std_rnn, std_gru, std_lstm, std_mlp) it can be specified using the -t flag:
+
+``test/npu_test.sh -t {test}``
+
+NPU Tests rely on a baseline file located in the ``rad-sim/example-designs/npu/scripts/perf_baseline`` directory.
+This file provides expected QoR and runtime values for each test case. The format of this file is ``{test} {QoR} {runtime}``.
+
+Running the NPU test produces a correctness result (PASS/FAIL), along with a performance result (PASS/FAIL).
+The performance result requires both the QoR value and the runtime to fall within specified thresholds from the baseline.
+These thresholds are defined in the ``rad-sim/example-designs/npu/scripts/perf_tests.py`` file.

rad-sim/example-designs/npu/scripts/perf_baseline

@@ -1,18 +1,18 @@
-01_gemv_512x512 3.93
-02_gemv_1024x1024 10.43
-03_gemv_1152x1152 11.11
-04_gemv_1536x1536 15.85
-05_gemv_1792x1792 17.79
-06_rnn_512_8 8.00
-07_rnn_1024_8 20.81
-08_rnn_1152_8 22.12
-09_rnn_1536_8 30.96
-10_rnn_1792_8 34.37
-11_gru_512_8 8.67
-12_gru_1024_8 21.34
-13_gru_1152_8 23.34
-14_lstm_512_8 13.56
-15_lstm_1024_8 30.01
-16_mlp5_512 4.73
-17_mlp5_1024 12.87
-18_mlp3_dlrm 4.21
+01_std_gemv_512x512 5.38 47.09
+02_std_gemv_1024x1024 12.47 80.72
+03_std_gemv_1152x1152 13.34 96.12
+04_std_gemv_1536x1536 19.62 121.48
+05_std_gemv_1792x1792 23.02 140.83
+06_std_rnn_512_8 7.44 295.92
+07_std_rnn_1024_8 17.42 550.32
+08_std_rnn_1152_8 19.35 629.05
+09_std_rnn_1536_8 27.75 831.69
+10_std_rnn_1792_8 33.13 999.86
+11_std_gru_512_8 8.6 979.34
+12_std_gru_1024_8 19.71 1863.19
+13_std_gru_1152_8 23.57 2042.29
+14_std_lstm_512_8 10.14 827.23
+15_std_lstm_1024_8 23.87 1526.99
+16_std_mlp5_512 6.81 60.49
+17_std_mlp5_1024 15.92 110.30
+18_std_mlp3_dlrm 5.38 68.64

rad-sim/example-designs/npu/scripts/perf_tests.py

@@ -20,6 +20,8 @@ class colors:
 vrf_depth = 512
 mrf_depth = 1024
 instances = 1
+qor_tolerance = 10.00 # 10 percent
+runtime_tolerance = 50.00 # 50 percent
 
 # Parse command line arguments
 if('-t' in sys.argv):
@@ -122,13 +124,14 @@ class colors:
 baseline = open('../scripts/perf_baseline', 'r')
 for line in baseline:
 	split_line = line.split(' ')
-	baseline_results[split_line[0]] = float(split_line[1])
+	baseline_results[split_line[0]] = (float(split_line[1]), float(split_line[2]))
 
-print(colors.BOLD + '{:<35}    {:<4}    {:<5}    {:<7}    {:<8}    {:<11}    {:<20}'.format('WORKLOAD', 'TEST', 'TOPS', 'QoR', 'Cycles', 'Runtime(s)', 'Speed(s/cyc)') + colors.RESET)
+print(colors.BOLD + '{:<35}    {:<4}    {:<4}    {:<5}    {:<7}    {:<8}    {:<11}    {:<20}'.format('WORKLOAD', 'TEST', 'PERF', 'TOPS', 'QoR', 'Cycles', 'Runtime(s)', 'Speed(s/cyc)') + colors.RESET)
 
 chdir('../compiler')
 first_flag = True
 cycles = ''
+global_failed = False
 for workload in workloads:
 	subprocess.call(['cp', '../scripts/workloads/'+workload+'.py', './'], shell=False)
 	sys.stdout.write('{:<35}    '.format(workload))
@@ -146,33 +149,59 @@ class colors:
 		subprocess.call(call_args, stdout=outfile, stderr=outfile, shell=False)
 	rptfile = open('../scripts/reports/'+workload+'_perf.rpt', 'r')
 	parse_perf_res = False
+
+	# Parse File
+	tops = 0
+	cycles = 0
+	runtime = 0
+	speed = 0
+	correctness = False
 	for line in rptfile:
 		if (parse_perf_res and ('Running simulation ... ' in line)):
 			args = line.split()
 			if('PASSED' in args[3]):
-				print(colors.PASS + 'PASS' + colors.RESET, end='    ')
 				result = args[10]
-				cycles = args[4][1:]
-				if workload in baseline_results:
-					comparison_to_baseline = ((float(args[10]) * instances/baseline_results[workload])-1) * 100
-					if comparison_to_baseline >= 0:
-						print ('{:<5}    +{:<5.2f}%    {:<8}    '.format(float(result) * instances, comparison_to_baseline, int(cycles)), end='')
-					else:
-						print ('{:<5}     {:<5.2f}%    {:<8}    '.format(float(result) * instances, comparison_to_baseline, int(cycles)), end='')
-				else:
-					print ('{:<5}    {:<7}    {:<8}    '.format(float(result) * instances, 'N/A', int(cycles)), end='')
+				tops = float(result) * instances
+				cycles = int(args[4][1:])
+				correctness = True	
 			else:
-				print(colors.FAIL + 'FAIL' + colors.RESET)
+				correctness = False
 		elif (parse_perf_res and ('Simulation took' in line)):
 			args = line.split()
 			args = args[2].split('m')
 			runtime = int(args[0]) * 60.0
 			args = args[1].split('s')
 			runtime = runtime + float(args[0])
-			print('{:<11.2f}    {:<20.2f}'.format(runtime, runtime/int(cycles)))	
+			speed = runtime/cycles
 		elif 'SystemC Performance Simulation' in line:
 			parse_perf_res = True
+
+	# Write Results
 	if(not parse_perf_res):
 		print(colors.FAIL + 'FAIL' + colors.RESET)
+		global_failed = True
+	else:
+		if(correctness):
+			print(colors.PASS + 'PASS' + colors.RESET, end='    ') # Correctness PASS
+			if workload in baseline_results:
+				qor_comparison_to_baseline = ((tops/baseline_results[workload][0])-1) * 100
+				runtime_comparison_to_baseline = ((runtime/baseline_results[workload][1])-1) * 100
+				if (abs(qor_comparison_to_baseline) < qor_tolerance and abs(runtime_comparison_to_baseline) < runtime_tolerance):
+					print(colors.PASS + 'PASS' + colors.RESET, end='    ') # Performance PASS
+				else:
+					print(colors.FAIL + 'FAIL' + colors.RESET, end='    ') # Performance FAIL
+					global_failed = True
+				if qor_comparison_to_baseline >= 0:
+					print ('{:<5}    +{:<5.2f}%    {:<8}    {:<11.2f}    {:<20.2f}'.format(tops, qor_comparison_to_baseline, cycles, runtime, speed))
+				else:
+					print ('{:<5}     {:<5.2f}%    {:<8}    {:<11.2f}    {:<20.2f}'.format(tops, qor_comparison_to_baseline, cycles, runtime, speed))
+			else:
+				print ('    ', end='    ') # No Performance Result
+				print ('{:<5}    {:<7}    {:<8}    {:<11.2f}    {:<20.2f}'.format(tops, 'N/A', cycles, runtime, speed))
+		else:
+			print(colors.FAIL + 'FAIL' + colors.RESET) # Correctness FAIL
+			global_failed = True
+
 	subprocess.call(['rm', workload+'.py'], shell=False)
 
+exit(global_failed)