feat(sast): Remove semgrep and turn CDK UTs into integration tests (bridgecrewio#81)

* remove parser tests

* move cdk test files

* move cdk test files 2

* remove parsers

* update registries (cdk + sast)

* remove sast consts

* remove semgrep dependency

* update prisma engine

* cdk tests v1 -> v2

* remove semgrep engine

* update sast report check

* remove old tests

* update runner + tests + report

* new integration tests

* initial cdk tests

* remove pipfile script

* cdk integration tests step

* cdk integration tests step fix

* cdk integration tests step fix

* cdk update checks

* cdk update checks

* cdk update checks

* remove old cdk tests

* fix tests

* remove unused mypy comment

* remove old functions from base_registry.py

* improve integration tests bash

* typing

* lint

* lint

* remove semgrep leftovers

* update prepare_data.sh

* bql v1 -> v2

* skip tests

* move engine to self

* update bash script

* support remove_default_policies flag

* test cdk runner through integration tests, using cdk as framework

* make cdk run all langs

* 0.1 to 0.2

* lint

* lint

* import fix

* import fix

* lint

---------

Co-authored-by: Rotem Avni <[email protected]>

Author: arielkru

.flake8

@@ -5,7 +5,7 @@ max-line-length = 120
 # E203,E501 don't work with black together
 ignore = E203,E501,E731,W503,W504,DUO107,DUO104,DUO130,DUO109,DUO116,B028,B950,TC001,TC003,TC006,B907
 select = C,E,F,W,B,B9,A,TC
-extend-exclude = .github, .pytest_cache, docs/*, venv/*, tests/*, flake8_plugins/*
+extend-exclude = .github, .pytest_cache, docs/*, venv/*, tests/*, flake8_plugins/*, cdk_integration_tests/src/python/*
 
 [flake8:local-plugins]
 extension =

.github/workflows/pr-test.yml

@@ -161,6 +161,44 @@ jobs:
         run: |
           pipenv run pytest sast_integration_tests
 
+  cdk-integration-tests:
+    strategy:
+      fail-fast: true
+      matrix:
+        python: [ "3.8" ]
+        os: [ ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab  # v3
+      - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0  # v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pipenv"
+          cache-dependency-path: "Pipfile.lock"
+      - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c  # v3
+      - name: Install pipenv
+        run: |
+          python -m pip install --no-cache-dir --upgrade pipenv
+      - name: Build & install checkov package
+        run: |
+          # remove venv, if exists
+          pipenv --rm || true
+          pipenv --python ${{ matrix.python }}
+          pipenv run pip install pytest pytest-xdist
+          pipenv run python setup.py sdist bdist_wheel
+          bash -c 'pipenv run pip install dist/checkov3-*.whl'
+      - name: Create checkov reports
+        env:
+          LOG_LEVEL: INFO
+          BC_API_KEY: ${{ secrets.BC_API_KEY }}
+        run: bash -c './cdk_integration_tests/prepare_data.sh'
+      - name: Run integration tests
+        env:
+          LOG_LEVEL: INFO
+          BC_API_KEY: ${{ secrets.BC_API_KEY }}
+        run: |
+          pipenv run pytest cdk_integration_tests
+
   performance-tests:
     if:
       true == false

Pipfile

@@ -89,7 +89,6 @@ yarl = "*"
 openai = "*"
 spdx-tools = "<0.8.0"
 license-expression = "==30.1.0"
-semgrep = "==1.10.0"
 pydantic = "==1.10.7"
 
 [requires]

checkov/sast/checks/__init__.py renamed to cdk_integration_tests/__init__.py

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# iterate over all the cdk python checks
+for file in "checkov/cdk/checks/python"/*; do
+  # Ensure it's a yaml file
+    if [[ -f "$file" && "$file" == *.yaml ]]; then
+        basename=$(basename -- "$file")
+        filename="${basename%.*}"
+        # create a report for this check
+        echo "creating report for check: $filename"
+        pipenv run checkov -s --framework cdk -o json \
+          -d "cdk_integration_tests/src/python/$filename" \
+          --external-checks-dir "checkov/cdk/checks/python/$filename.yaml" > "checkov_report_cdk_python_$filename.json"
+    fi
+done
+
+#todo: iterate over all the cdk typescript checks - when ts supported in sast

cdk_integration_tests/prepare_data.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# In order to run this script set the following environment variables:
+# BC_API_URL - your API url.
+# BC_KEY - generate API key via Platform.
+# You can also add the local SAST_ARTIFACT_PATH and LOG_LEVEL.
+
+# You can also set those vars in the set_env_vars() function, and uncomment the call to it.
+
+# The working dir should be the checkov project dir.
+# For example: on /Users/ajbara/dev2/checkov dir run BC_API_URL=https://ws342vj2ze.execute-api.us-west-2.amazonaws.com/v1 BC_KEY=xyz LOG_LEVEL=Info /Users/ajbara/dev2/checkov/sast_integration_tests/run_integration_tests.sh
+
+set_env_vars() {
+  export SAST_ARTIFACT_PATH=""
+  export BC_API_KEY=""
+  export LOG_LEVEL=DEBUG
+  export BC_API_URL=""
+}
+
+prepare_data () {
+  for file in "checkov/cdk/checks/python"/*; do
+  # Ensure it's a regular file (not a directory or symlink, etc.)
+    if [ -f "$file" ]; then
+        basename=$(basename -- "$file")
+        filename="${basename%.*}"
+        # create a report for this check
+        echo "creating report for check: $filename"
+        python checkov/main.py -s --framework cdk -o json \
+          -d "cdk_integration_tests/src/python/$filename" \
+          --external-checks-dir "checkov/cdk/checks/python/$filename.yaml" > "checkov_report_cdk_python_$filename.json"
+    fi
+done
+
+}
+
+delete_reports () {
+  rm -r checkov_report*
+  rm results.sarif
+  rm checkov_checks_list.txt
+}
+
+echo "calling set_env_vars"
+set_env_vars
+
+if [[ -z "BC_API_KEY" ]]; then
+   echo "BC_API_KEY is missing."
+   exit 1
+fi
+
+echo $BC_API_URL
+if [[ -z "$BC_API_URL" ]]; then
+   echo "BC_API_URL is missing."
+   exit 1
+fi
+
+cd ..
+
+echo $VIRTUAL_ENV
+if [ ! -z "$VIRTUAL_ENV" ]; then
+  deactivate
+fi
+
+#activate virtual env
+ENV_PATH=$(pipenv --venv)
+echo $ENV_PATH
+source $ENV_PATH/bin/activate
+
+echo $(pwd)
+working_dir=$(pwd) # should be the path of local checkov project
+export PYTHONPATH="$working_dir/checkov:$PYTHONPATH"
+
+prepare_data
+
+#Run integration tests.
+echo "running integration tests"
+pytest cdk_integration_tests
+
+deactivate
+
+echo "Deleting reports"
+delete_reports
+