From a0a4ba5a81a7290368dfe0d3c1221e5d01bf62fc Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 5 Sep 2024 14:25:37 +0000 Subject: [PATCH 1/7] Add sbom generation --- .github/workflows/trivy.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 27666359..a90951f8 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -3,12 +3,13 @@ on: push: branches: - 3.4-22.04/edge + - dpe-5350-3.4 # tmp to test new action. pull_request: jobs: build: uses: ./.github/workflows/build.yaml scan: - name: Trivy scan + name: Trivy scan and sbom generation needs: build runs-on: ubuntu-20.04 steps: @@ -46,4 +47,20 @@ jobs: uses: github/codeql-action/upload-sarif@v2 if: always() with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file + sarif_file: 'trivy-results.sarif' + + - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: 'fs' + format: 'github' + output: 'dependency-results.sbom.json' + image-ref: 'trivy/charmed-spark:test' + github-pat: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload trivy report as a Github artifact + uses: actions/upload-artifact@v4 + with: + name: trivy-sbom-report + path: '${{ github.workspace }}/dependency-results.sbom.json' + retention-days: 90 From 7d22375670987e7eff3356904491c728a7139b70 Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 5 Sep 2024 14:44:51 +0000 Subject: [PATCH 2/7] Update action to scan image --- .github/workflows/trivy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index a90951f8..1af0ed16 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -52,11 +52,13 @@ jobs: - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph uses: aquasecurity/trivy-action@0.20.0 with: - scan-type: 'fs' + scan-type: 'image' format: 'github' output: 'dependency-results.sbom.json' image-ref: 'trivy/charmed-spark:test' github-pat: ${{ secrets.GITHUB_TOKEN }} + severity: "MEDIUM,HIGH,CRITICAL" + scanners: "vuln" - name: Upload trivy report as a Github artifact uses: actions/upload-artifact@v4 From bc8e4797b15869c62caf240d62e40774d8065014 Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Wed, 11 Sep 2024 15:04:56 +0000 Subject: [PATCH 3/7] switch to spdx format --- .github/workflows/trivy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1af0ed16..03323479 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -53,7 +53,7 @@ jobs: uses: aquasecurity/trivy-action@0.20.0 with: scan-type: 'image' - format: 'github' + format: 'spdx' output: 'dependency-results.sbom.json' image-ref: 'trivy/charmed-spark:test' github-pat: ${{ secrets.GITHUB_TOKEN }} From 772dcc23a0a2fec265fe2293d8dc7bed10ff6a2c Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 12 Sep 2024 12:19:58 +0000 Subject: [PATCH 4/7] Fix minio problems --- .github/workflows/trivy.yml | 2 +- tests/integration/setup-aws-cli.sh | 9 +++------ tests/integration/utils/s3-utils.sh | 24 ++++++++++++++++++++---- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 03323479..b1b02a08 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -53,7 +53,7 @@ jobs: uses: aquasecurity/trivy-action@0.20.0 with: scan-type: 'image' - format: 'spdx' + format: 'spdx-json' output: 'dependency-results.sbom.json' image-ref: 'trivy/charmed-spark:test' github-pat: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/integration/setup-aws-cli.sh b/tests/integration/setup-aws-cli.sh index a6386697..5f513d35 100755 --- a/tests/integration/setup-aws-cli.sh +++ b/tests/integration/setup-aws-cli.sh @@ -4,11 +4,8 @@ sudo snap install aws-cli --classic -get_s3_endpoint(){ - # Get S3 endpoint from MinIO - kubectl get service minio -n minio-operator -o jsonpath='{.spec.clusterIP}' -} +source ./utils/s3-utils.sh wait_and_retry(){ # Retry a command for a number of times by waiting a few seconds. @@ -37,8 +34,8 @@ wait_and_retry get_s3_endpoint S3_ENDPOINT=$(get_s3_endpoint) DEFAULT_REGION="us-east-2" -ACCESS_KEY=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) -SECRET_KEY=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d) +ACCESS_KEY=$(get_s3_access_key) +SECRET_KEY=$(get_s3_secret_key) # Configure AWS CLI credentials aws configure set aws_access_key_id $ACCESS_KEY diff --git a/tests/integration/utils/s3-utils.sh b/tests/integration/utils/s3-utils.sh index 9008f0e8..c297e83d 100644 --- a/tests/integration/utils/s3-utils.sh +++ b/tests/integration/utils/s3-utils.sh @@ -20,14 +20,30 @@ get_s3_endpoint(){ get_s3_access_key(){ - # Print the S3 Access Key by reading it from K8s secret - kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d + # Print the S3 Access Key by reading it from K8s secret or by outputting the default value + kubectl get secret -n minio-operator microk8s-user-1 + if [ $? -eq 0 ]; then + echo "Use access-key from secret" + access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) + else + echo "use default access-key" + access_key="minio" + fi + echo "$access_key" } get_s3_secret_key(){ - # Print the S3 Secret Key by reading it from K8s secret - kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d + # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value + kubectl get secret -n minio-operator microk8s-user-1 + if [ $? -eq 0 ]; then + echo "Use access-key from secret" + secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d) + else + echo "use default access-key" + secret_key="minio123" + fi + echo "$secret_key" } From 40966da6d19a3baa2cb097ee03bd2ce2d6be97a5 Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 12 Sep 2024 14:34:05 +0000 Subject: [PATCH 5/7] try to fix tests --- tests/integration/setup-aws-cli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/setup-aws-cli.sh b/tests/integration/setup-aws-cli.sh index 5f513d35..f69932fe 100755 --- a/tests/integration/setup-aws-cli.sh +++ b/tests/integration/setup-aws-cli.sh @@ -5,7 +5,7 @@ sudo snap install aws-cli --classic -source ./utils/s3-utils.sh +source ./tests/integration/utils/s3-utils.sh wait_and_retry(){ # Retry a command for a number of times by waiting a few seconds. From 5ef54d73cbae493e391548e191699a4ed6fbedad Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 12 Sep 2024 15:25:06 +0000 Subject: [PATCH 6/7] try to fix tests --- tests/integration/setup-aws-cli.sh | 32 +++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/integration/setup-aws-cli.sh b/tests/integration/setup-aws-cli.sh index f69932fe..ee1deacc 100755 --- a/tests/integration/setup-aws-cli.sh +++ b/tests/integration/setup-aws-cli.sh @@ -4,8 +4,38 @@ sudo snap install aws-cli --classic +get_s3_endpoint(){ + # Print the endpoint where the S3 bucket is exposed on. + kubectl get service minio -n minio-operator -o jsonpath='{.spec.clusterIP}' +} + + +get_s3_access_key(){ + # Print the S3 Access Key by reading it from K8s secret or by outputting the default value + kubectl get secret -n minio-operator microk8s-user-1 + if [ $? -eq 0 ]; then + echo "Use access-key from secret" + access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) + else + echo "use default access-key" + access_key="minio" + fi + echo "$access_key" +} + -source ./tests/integration/utils/s3-utils.sh +get_s3_secret_key(){ + # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value + kubectl get secret -n minio-operator microk8s-user-1 + if [ $? -eq 0 ]; then + echo "Use access-key from secret" + secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d) + else + echo "use default access-key" + secret_key="minio123" + fi + echo "$secret_key" +} wait_and_retry(){ # Retry a command for a number of times by waiting a few seconds. From 6d32220ec0aa45c5cbb0fbe9141dcc08d0da2603 Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Thu, 12 Sep 2024 15:56:23 +0000 Subject: [PATCH 7/7] Fix test --- tests/integration/setup-aws-cli.sh | 18 +++++++++--------- tests/integration/utils/s3-utils.sh | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/integration/setup-aws-cli.sh b/tests/integration/setup-aws-cli.sh index ee1deacc..9966ffd2 100755 --- a/tests/integration/setup-aws-cli.sh +++ b/tests/integration/setup-aws-cli.sh @@ -2,7 +2,7 @@ # Install AWS CLI sudo snap install aws-cli --classic - +set -x get_s3_endpoint(){ # Print the endpoint where the S3 bucket is exposed on. @@ -12,13 +12,13 @@ get_s3_endpoint(){ get_s3_access_key(){ # Print the S3 Access Key by reading it from K8s secret or by outputting the default value - kubectl get secret -n minio-operator microk8s-user-1 + kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null if [ $? -eq 0 ]; then - echo "Use access-key from secret" - access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) + # echo "Use access-key from secret" + access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) else - echo "use default access-key" - access_key="minio" + # echo "use default access-key" + access_key="minio" fi echo "$access_key" } @@ -26,12 +26,12 @@ get_s3_access_key(){ get_s3_secret_key(){ # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value - kubectl get secret -n minio-operator microk8s-user-1 + kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null if [ $? -eq 0 ]; then - echo "Use access-key from secret" + # echo "Use access-key from secret" secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d) else - echo "use default access-key" + # echo "use default access-key" secret_key="minio123" fi echo "$secret_key" diff --git a/tests/integration/utils/s3-utils.sh b/tests/integration/utils/s3-utils.sh index c297e83d..7c318aba 100644 --- a/tests/integration/utils/s3-utils.sh +++ b/tests/integration/utils/s3-utils.sh @@ -21,13 +21,13 @@ get_s3_endpoint(){ get_s3_access_key(){ # Print the S3 Access Key by reading it from K8s secret or by outputting the default value - kubectl get secret -n minio-operator microk8s-user-1 + kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null if [ $? -eq 0 ]; then - echo "Use access-key from secret" - access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) + # echo "Use access-key from secret" + access_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_ACCESS_KEY}' | base64 -d) else - echo "use default access-key" - access_key="minio" + # echo "use default access-key" + access_key="minio" fi echo "$access_key" } @@ -35,12 +35,12 @@ get_s3_access_key(){ get_s3_secret_key(){ # Print the S3 Secret Key by reading it from K8s secret or by outputting the default value - kubectl get secret -n minio-operator microk8s-user-1 + kubectl get secret -n minio-operator microk8s-user-1 &> /dev/null if [ $? -eq 0 ]; then - echo "Use access-key from secret" + # echo "Use access-key from secret" secret_key=$(kubectl get secret -n minio-operator microk8s-user-1 -o jsonpath='{.data.CONSOLE_SECRET_KEY}' | base64 -d) else - echo "use default access-key" + # echo "use default access-key" secret_key="minio123" fi echo "$secret_key"