From 23209c46f2b50c61bbad9e6b0584482ff6f1be89 Mon Sep 17 00:00:00 2001 From: Jou Ho <43765840+jouho@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:35:37 -0800 Subject: [PATCH] ci: run fuzz tests in parallel and generate coverage report (#4960) --- CMakeLists.txt | 19 +-- codebuild/bin/fuzz_corpus_download.sh | 33 +++++ codebuild/bin/fuzz_corpus_upload.sh | 25 ++++ codebuild/bin/fuzz_coverage_report.sh | 75 ++++++++++ codebuild/spec/buildspec_fuzz.yml | 2 +- codebuild/spec/buildspec_fuzz_scheduled.yml | 69 +++------ tests/fuzz/Readme.md | 40 ++++-- tests/fuzz/calcTotalCov.sh | 59 -------- tests/fuzz/runFuzzTest.sh | 147 +++----------------- 9 files changed, 200 insertions(+), 269 deletions(-) create mode 100755 codebuild/bin/fuzz_corpus_download.sh create mode 100755 codebuild/bin/fuzz_corpus_upload.sh create mode 100755 codebuild/bin/fuzz_coverage_report.sh delete mode 100755 tests/fuzz/calcTotalCov.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index f19e25d1440..a56871fbece 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -246,7 +246,7 @@ if (COVERAGE) # on LLVM compilers. GCC would fail with "unrecognized compile options" # on -fprofile-instr-generate -fcoverage-mapping flags. if (NOT ${CMAKE_C_COMPILER_ID} MATCHES Clang) - message(FATAL_ERROR "This project requires clang for coverage support") + message(FATAL_ERROR "This project requires clang for coverage support. You are currently using " ${CMAKE_C_COMPILER_ID}) endif() target_compile_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping) target_link_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping) @@ -667,7 +667,6 @@ if (BUILD_TESTING) if(S2N_FUZZ_TEST) message(STATUS "Fuzz build enabled") set(SCRIPT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/runFuzzTest.sh") - set(BUILD_DIR_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build") file(GLOB FUZZ_TEST_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/*.c") file(GLOB TESTLIB_SRC "tests/testlib/*.c") @@ -684,18 +683,6 @@ if (BUILD_TESTING) set(FUZZ_TIMEOUT_SEC 60) endif() - if(DEFINED ENV{CORPUS_UPLOAD_LOC}) - set(CORPUS_UPLOAD_LOC $ENV{CORPUS_UPLOAD_LOC}) - else() - set(CORPUS_UPLOAD_LOC "none") - endif() - - if(DEFINED ENV{ARTIFACT_UPLOAD_LOC}) - set(ARTIFACT_UPLOAD_LOC $ENV{ARTIFACT_UPLOAD_LOC}) - else() - set(ARTIFACT_UPLOAD_LOC "none") - endif() - # Build LD_PRELOAD shared libraries file(GLOB LIBRARY_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/LD_PRELOAD/*.c") foreach(SRC ${LIBRARY_SRCS}) @@ -729,9 +716,7 @@ if (BUILD_TESTING) bash ${SCRIPT_PATH} ${TEST_NAME} ${FUZZ_TIMEOUT_SEC} - ${BUILD_DIR_PATH} - ${CORPUS_UPLOAD_LOC} - ${ARTIFACT_UPLOAD_LOC} + ${CMAKE_CURRENT_SOURCE_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz ) set_property(TEST ${TEST_NAME} PROPERTY LABELS "fuzz") diff --git a/codebuild/bin/fuzz_corpus_download.sh b/codebuild/bin/fuzz_corpus_download.sh new file mode 100755 index 00000000000..0da91abcbf6 --- /dev/null +++ b/codebuild/bin/fuzz_corpus_download.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +for FUZZ_TEST in tests/fuzz/*.c; do + # extract file name without extension + TEST_NAME=$(basename "$FUZZ_TEST") + TEST_NAME="${TEST_NAME%.*}" + + # temp corpus folder to store downloaded corpus files + TEMP_CORPUS_DIR="./tests/fuzz/temp_corpus_${TEST_NAME}" + + # Check if corpus.zip exists in the specified S3 location. + # `> /dev/null 2>&1` redirects output to /dev/null. + # If the file is not found, `aws s3 ls` returns a non-zero exit code. + if aws s3 ls "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" > /dev/null 2>&1; then + aws s3 cp "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" "${TEMP_CORPUS_DIR}/corpus.zip" + unzip -o "${TEMP_CORPUS_DIR}/corpus.zip" -d "${TEMP_CORPUS_DIR}" > /dev/null 2>&1 + else + printf "corpus.zip not found for ${TEST_NAME}" + fi +done diff --git a/codebuild/bin/fuzz_corpus_upload.sh b/codebuild/bin/fuzz_corpus_upload.sh new file mode 100755 index 00000000000..8100e074166 --- /dev/null +++ b/codebuild/bin/fuzz_corpus_upload.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +for FUZZ_TEST in tests/fuzz/*.c; do + # extract file name without extension + TEST_NAME=$(basename "$FUZZ_TEST") + TEST_NAME="${TEST_NAME%.*}" + + # Upload generated corpus files to the S3 bucket. + zip -r ./tests/fuzz/corpus/${TEST_NAME}.zip ./tests/fuzz/corpus/${TEST_NAME}/ > /dev/null 2>&1 + aws s3 cp ./tests/fuzz/corpus/${TEST_NAME}.zip s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip +done + diff --git a/codebuild/bin/fuzz_coverage_report.sh b/codebuild/bin/fuzz_coverage_report.sh new file mode 100755 index 00000000000..a35e4e77d4e --- /dev/null +++ b/codebuild/bin/fuzz_coverage_report.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +set -e + +usage() { + echo "Usage: fuzz_coverage_report.sh" + exit 1 +} + +if [ "$#" -ne "0" ]; then + usage +fi + +FUZZ_TEST_DIR="tests/fuzz" +FUZZCOV_SOURCES="api bin crypto error stuffer tls utils" + +# generate coverage report for each fuzz test +printf "Generating coverage reports... \n" + +mkdir -p coverage/fuzz +for FUZZ_TEST in "$FUZZ_TEST_DIR"/*.c; do + # extract file name without extension + TEST_NAME=$(basename "$FUZZ_TEST") + TEST_NAME="${TEST_NAME%.*}" + + # merge multiple .profraw files into a single .profdata file + llvm-profdata merge \ + -sparse tests/fuzz/profiles/${TEST_NAME}/*.profraw \ + -o tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata + + # generate a coverage report in text format + llvm-cov report \ + -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ + -show-functions \ + > coverage/fuzz/${TEST_NAME}_cov.txt + + # exports coverage data in LCOV format + llvm-cov export \ + -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ + -format=lcov \ + > coverage/fuzz/${TEST_NAME}_cov.info + + # convert to HTML format + genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info > /dev/null 2>&1 +done + +# merge all coverage reports into a single report that shows total s2n coverage +printf "Calculating total s2n coverage... \n" +llvm-profdata merge \ + -sparse tests/fuzz/profiles/*/*.profdata \ + -o tests/fuzz/profiles/merged_fuzz.profdata + +llvm-cov report \ + -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ + > s2n_fuzz_coverage.txt + +llvm-cov export \ + -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ + -format=lcov \ + > s2n_fuzz_cov.info + +genhtml s2n_fuzz_cov.info --branch-coverage -q -o coverage/fuzz/total_fuzz_coverage diff --git a/codebuild/spec/buildspec_fuzz.yml b/codebuild/spec/buildspec_fuzz.yml index c74700ff28f..55faa40afdc 100644 --- a/codebuild/spec/buildspec_fuzz.yml +++ b/codebuild/spec/buildspec_fuzz.yml @@ -33,4 +33,4 @@ phases: on-failure: ABORT commands: # -L: Restrict tests to names matching the pattern 'fuzz' - - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure" + - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc)" diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml index b7672a4e2c9..29c9c8b354e 100644 --- a/codebuild/spec/buildspec_fuzz_scheduled.yml +++ b/codebuild/spec/buildspec_fuzz_scheduled.yml @@ -13,55 +13,10 @@ # limitations under the License. version: 0.2 -batch: - build-matrix: - static: - env: - privileged-mode: true - dynamic: - env: - compute-type: - - BUILD_GENERAL1_LARGE - image: - - 024603541914.dkr.ecr.us-west-2.amazonaws.com/docker:ubuntu22codebuild - privileged-mode: true - variables: - S2N_LIBCRYPTO: - - awslc - FUZZ_TESTS: - - "s2n_cert_req_recv_test" - - "s2n_certificate_extensions_parse_test" - - "s2n_client_ccs_recv_test" - - "s2n_client_cert_recv_test" - - "s2n_client_cert_verify_recv_test" - - "s2n_client_finished_recv_test" - - "s2n_client_fuzz_test" - - "s2n_client_hello_recv_fuzz_test" - - "s2n_client_key_recv_fuzz_test" - - "s2n_deserialize_resumption_state_test" - - "s2n_encrypted_extensions_recv_test" - - "s2n_extensions_client_key_share_recv_test" - - "s2n_extensions_client_supported_versions_recv_test" - - "s2n_extensions_server_key_share_recv_test" - - "s2n_extensions_server_supported_versions_recv_test" - - "s2n_hybrid_ecdhe_kyber_r3_fuzz_test" - - "s2n_kyber_r3_recv_ciphertext_fuzz_test" - - "s2n_kyber_r3_recv_public_key_fuzz_test" - - "s2n_memory_leak_negative_test" - - "s2n_openssl_diff_pem_parsing_test" - - "s2n_recv_client_supported_groups_test" - - "s2n_select_server_cert_test" - - "s2n_server_ccs_recv_test" - - "s2n_server_cert_recv_test" - - "s2n_server_extensions_recv_test" - - "s2n_server_finished_recv_test" - - "s2n_server_fuzz_test" - - "s2n_server_hello_recv_test" - - "s2n_stuffer_pem_fuzz_test" - - "s2n_tls13_cert_req_recv_test" - - "s2n_tls13_cert_verify_recv_test" - - "s2n_tls13_client_finished_recv_test" - - "s2n_tls13_server_finished_recv_test" +env: + variables: + S2N_LIBCRYPTO: "awslc" + COMPILER: clang phases: pre_build: @@ -76,13 +31,23 @@ phases: - | cmake . -Bbuild \ -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \ + -DCMAKE_C_COMPILER=/usr/bin/$COMPILER \ -DS2N_FUZZ_TEST=on \ - -DFUZZ_TIMEOUT_SEC=27000 + -DCOVERAGE=on \ + -DBUILD_SHARED_LIBS=on - cmake --build ./build -- -j $(nproc) post_build: on-failure: ABORT commands: + - ./codebuild/bin/fuzz_corpus_download.sh # -L: Restrict tests to labels matching the pattern 'fuzz' - # -R: Run the single fuzz test defined in ${FUZZ_TESTS} # --timeout: override ctest's default timeout of 1500 - - cmake --build build/ --target test -- ARGS="-L fuzz -R ${FUZZ_TESTS} --output-on-failure --timeout 28800" + - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc) --timeout 28800" + - ./codebuild/bin/fuzz_corpus_upload.sh + - ./codebuild/bin/fuzz_coverage_report.sh + +artifacts: + # upload all files in the fuzz_coverage_report directory + files: + - '**/*' + base-directory: coverage/fuzz/total_fuzz_coverage diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md index 28e4bb2da3a..49fe930f39b 100644 --- a/tests/fuzz/Readme.md +++ b/tests/fuzz/Readme.md @@ -29,23 +29,39 @@ cmake --build build/ --target test -- ARGS="-L fuzz -R s2n_client_fuzz_test --ou 2. If the test ends with `*_negative_test.c` the test is expected to fail in some way or return a non-zero integer (hereafter referred to as a "Negative test"). 2. Strive to be deterministic (Eg. shouldn't depend on the time or on the output of a RNG). Each test should either always pass if a Positive Test, or always fail if a Negative Test. 3. If a Positive Fuzz test, it should have a non-empty corpus directory with inputs that have a relatively high branch coverage. -4. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup. -5. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's -5. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state. -6. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target +4. If a Positive Fuzz test, define target functions for the test by adding following lines to your test below the copyright notice: +> /* Target Functions: function1 function2 function3 */ +5. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup. +6. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's +7. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state. +8. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target ## Fuzz Test Coverage -To generate coverage reports for fuzz tests, simply set the FUZZ_COVERAGE environment variable to any non-null value and run `make fuzz`. This will report the target function coverage and overall S2N coverage when running the tests. In order to define target functions for a fuzz test, simply add the following line to your fuzz test below the copyright notice: +We run fuzz tests daily, with corpus files that are continuously being improved. Current coverage information can be view [here](https://dx1inn44oyl7n.cloudfront.net/fuzz-coverage-report/index.html). -> /* Target Functions: function1 function2 function3 */ +To generate coverage reports for fuzz tests, s2n-tls needs to be compiled with the following options: +``` +cmake . -Bbuild \ +-DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \ +-DS2N_FUZZ_TEST=on \ +-DCOVERAGE=on \ +-DBUILD_SHARED_LIBS=on -As the tests run, more detailed coverage reports are placed in the following directory: +cmake --build ./build -- -j $(nproc) +``` -> s2n/coverage/fuzz +Next, run fuzz tests. This generates `.info` files for each fuzz test containing coverage information. +``` +cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure" +``` -Each test outputs an HTML file which displays line by line coverage statistics and a .txt report which gives per-function coverage statistics in human-readable ASCII. After all fuzz tests have ran, a matching pair of coverage reports is generated for the total coverage of S2N by the entire set of tests performed. +The `.info` files contain raw coverage data. To convert them into HTML format, run the following script from the root of s2n-tls. This generates HTML files showing line-by-line coverage statistics for each fuzz test, as well as total coverage report for s2n-tls across all tests. +``` +./codebuild/bin/fuzz_coverage_report.sh +``` -Currently, this option isn't enabled for cmake build. See [#4748](https://github.com/aws/s2n-tls/issues/4748). +You will see coverage reports placed in the following directory: +> s2n-tls/tests/fuzz/coverage ## Fuzz Test Directory Structure For a test with name `$TEST_NAME`, its files should be laid out with the following structure: @@ -62,9 +78,7 @@ For a test with name `$TEST_NAME`, its files should be laid out with the followi # Corpus A Corpus is a directory of "interesting" inputs that result in a good branch/code coverage. These inputs will be permuted in random ways and checked to see if this permutation results in greater branch coverage or in a failure (Segfault, Memory Leak, Buffer Overflow, Non-zero return code, etc). If the permutation results in greater branch coverage, then it will be added to the Corpus directory. If a Memory leak or a Crash is detected, that file will **not** be added to the corpus for that test, and will instead be written to the current directory (`s2n/tests/fuzz/crash-*` or `s2n/tests/fuzz/leak-*`). These files will be automatically deleted for any Negative Fuzz tests that are expected to crash or leak memory so as to not clutter the directory. -To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing branch coverage over time. This process allows for gradual and automated enhancement of the corpus. - -To enable this, two environment variables must be defined: `CORPUS_UPLOAD_LOC` and `ARTIFACT_UPLOAD_LOC`. `CORPUS_UPLOAD_LOC` specifies where corpus files are stored, while `ARTIFACT_UPLOAD_LOC`defines where output logs from fuzzing are saved, which can be used for debugging if a new bug is detected during fuzzing. +To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing coverage over time. This process allows for gradual and automated enhancement of the corpus. # LD_PRELOAD The `LD_PRELOAD` directory contains function overrides for each Fuzz test that will be used **instead** of the original functions defined elsewhere. These function overrides will only be used during fuzz tests, and will not effect the rest of the s2n codebase when not fuzzing. Using `LD_PRELOAD` instead of C Preprocessor `#ifdef`'s is preferable in the following ways: diff --git a/tests/fuzz/calcTotalCov.sh b/tests/fuzz/calcTotalCov.sh deleted file mode 100755 index f36255e9b47..00000000000 --- a/tests/fuzz/calcTotalCov.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - -set -e - -usage() { - echo "Usage: calcTotalCov.sh" - exit 1 -} - -if [ "$#" -ne "0" ]; then - usage -fi - -if [[ -z "$S2N_ROOT" ]]; then - S2N_ROOT=../.. -fi - -FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils" - - -# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set -# Total coverage is overlayed on source code in s2n_cov.html and coverage statistics are available in s2n_cov.txt -# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML -# All files are stored in the s2n coverage directory -if [[ "$FUZZ_COVERAGE" == "true" ]]; then - - printf "Calculating total s2n coverage... " - - # The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but - # works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed. - llvm-profdata merge -sparse ./profiles/*/*.profdata -o ./profiles/s2n_cov.profdata > /dev/null 2>&1 - llvm-cov report -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/s2n_cov.txt - - # Use LCOV format instead of HTML if the LLVM version we're using supports it - if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) > 8 ]]; then - llvm-cov export -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/s2n_cov.info - genhtml -q -o ${COVERAGE_DIR}/html/overall_fuzz_coverage ${COVERAGE_DIR}/fuzz/s2n_cov.info - else - llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/s2n_cov.html - fi - # Generate coverage report compatible with codecov.io - llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/codecov.txt - - S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' ${COVERAGE_DIR}/fuzz/s2n_cov.txt | tail -1` - printf "total s2n coverage from fuzz tests: %s\n" $S2N_COV -fi diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh index 800135bca7b..1a70e797fb3 100755 --- a/tests/fuzz/runFuzzTest.sh +++ b/tests/fuzz/runFuzzTest.sh @@ -18,19 +18,18 @@ set -e usage() { - echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC" + echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC S2N_ROOT" exit 1 } -if [ "$#" -ne "5" ]; then +if [ "$#" -ne "3" ]; then usage fi TEST_NAME=$1 FUZZ_TIMEOUT_SEC=$2 -BUILD_DIR_PATH=$3 -CORPUS_UPLOAD_LOC=$4 -ARTIFACT_UPLOAD_LOC=$5 +S2N_ROOT=$3 + MIN_TEST_PER_SEC="1000" MIN_FEATURES_COVERED="100" @@ -42,14 +41,10 @@ else EXPECTED_TEST_FAILURE=0 fi -ASAN_OPTIONS+="symbolize=1" -LSAN_OPTIONS+="log_threads=1" -UBSAN_OPTIONS+="print_stacktrace=1" -NUM_CPU_THREADS=$(nproc) -LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -jobs=${NUM_CPU_THREADS} -workers=${NUM_CPU_THREADS} -max_total_time=${FUZZ_TIMEOUT_SEC}" +LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -max_total_time=${FUZZ_TIMEOUT_SEC}" -TEST_SPECIFIC_OVERRIDES="${BUILD_DIR_PATH}/lib/lib${TEST_NAME}_overrides.so" -GLOBAL_OVERRIDES="${BUILD_DIR_PATH}/lib/libglobal_overrides.so" +TEST_SPECIFIC_OVERRIDES="${S2N_ROOT}/build/lib/lib${TEST_NAME}_overrides.so" +GLOBAL_OVERRIDES="${S2N_ROOT}/build/lib/libglobal_overrides.so" FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils" @@ -79,40 +74,14 @@ ACTUAL_TEST_FAILURE=0 # Copy existing Corpus to a temp directory so that new inputs from fuzz tests runs will add new inputs to the temp directory. # This allows us to minimize new inputs before merging to the original corpus directory. -# If s3 directory is specified, use corpuses stored in S3 bucket instead. -TEMP_CORPUS_DIR="$(mktemp -d)" -if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then - ( - # Clean the environment before copying corpuses from the S3 bucket. - # The LD variables interferes with certificate validation when communicating with AWS S3. - unset LD_PRELOAD - unset LD_LIBRARY_PATH - - # Check if corpus.zip exists in the specified S3 location. - # `> /dev/null 2>&1` redirects output to /dev/null. - # If the file is not found, `aws s3 ls` returns a non-zero exit code. - if aws s3 ls "${CORPUS_UPLOAD_LOC}/${TEST_NAME}/corpus.zip" > /dev/null 2>&1; then - printf "corpus.zip found, downloading from S3 bucket and unzipping...\n" - aws s3 cp "${CORPUS_UPLOAD_LOC}/${TEST_NAME}/corpus.zip" "${TEMP_CORPUS_DIR}/corpus.zip" - unzip -o "${TEMP_CORPUS_DIR}/corpus.zip" -d "${TEMP_CORPUS_DIR}" - fi - ) -else - cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}" -fi +TEMP_CORPUS_DIR="temp_corpus_${TEST_NAME}" +cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}" -# Setup and clean profile structure if FUZZ_COVERAGE is enabled, otherwise run as normal -if [[ "$FUZZ_COVERAGE" == "true" ]]; then - mkdir -p "./profiles/${TEST_NAME}" - rm -f ./profiles/${TEST_NAME}/*.profraw - LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" \ - ${BUILD_DIR_PATH}/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \ - > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1 -else - env LD_PRELOAD="$LD_PRELOAD_" \ - ${BUILD_DIR_PATH}/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \ - > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1 -fi +# Run fuzz test executable and store results to an output file +env LD_PRELOAD="$LD_PRELOAD_" \ +LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" \ +${S2N_ROOT}/build/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \ +> ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1 TEST_INFO=$( grep -o "stat::number_of_executed_units: [0-9]*" ${TEST_NAME}_output.txt | \ @@ -120,68 +89,19 @@ TEST_INFO=$( ) TESTS_PER_SEC=$(echo "$TEST_INFO" | cut -d ' ' -f 3) FEATURE_COVERAGE=`grep -o "ft: [0-9]*" ${TEST_NAME}_output.txt | awk '{print $2}' | sort | tail -1` -TARGET_FUNCS='' -declare -i TARGET_TOTAL=0 -declare -i TARGET_COV=0 - -# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set -# Coverage is overlayed on source code in ${TEST_NAME}_cov.html, and coverage statistics are available in ${TEST_NAME}_cov.txt -# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML -# All files are stored in the s2n coverage directory -if [[ "$FUZZ_COVERAGE" == "true" ]]; then - mkdir -p ${COVERAGE_DIR}/fuzz - llvm-profdata merge -sparse ./profiles/${TEST_NAME}/*.profraw -o ./profiles/${TEST_NAME}/${TEST_NAME}.profdata - llvm-cov report -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -show-functions > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt - - # Use LCOV format instead of HTML if the LLVM version we're using supports it - if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) -gt 8 ]]; then - llvm-cov export -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info - genhtml -q -o ${COVERAGE_DIR}/html/${TEST_NAME} ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info - else - llvm-cov show -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.html - fi - - # Extract target functions from test source - TARGET_FUNCS=`grep -Pzo "(?<=/\* Target Functions: )[\w\s]*" ${TEST_NAME}.c | tr -d "\0"` - - # Find line coverage statistics for target functions - if [[ ! -z "$TARGET_FUNCS" ]]; - then - for TARGET in ${TARGET_FUNCS} - do - TARGET_TOTAL+=`sed -n "s/^.*${TARGET} .*% *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt` - TARGET_COV+=`sed -n "s/^.*${TARGET} .*% *[0-9]* *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt` - done - fi -fi if [ $ACTUAL_TEST_FAILURE == $EXPECTED_TEST_FAILURE ]; then - printf "\033[32;1mPASSED\033[0m %s" "$TEST_INFO" - - # Output target function coverage percentage if target functions are defined and fuzzing coverage is enabled - # Otherwise, print number of features covered - if [[ "$FUZZ_COVERAGE" == "true" && ! -z "$TARGET_FUNCS" && "$EXPECTED_TEST_FAILURE" != 1 && "$TARGET_TOTAL" != 0 ]]; - then - printf ", %6.2f%% target coverage" "$(( 10000 * ($TARGET_TOTAL - $TARGET_COV) / $TARGET_TOTAL ))e-2" - else - printf ", %5d features covered" $FEATURE_COVERAGE - fi - if [ $EXPECTED_TEST_FAILURE == 1 ]; then # Clean up LibFuzzer corpus files if the test is negative. - printf "\n" rm -f leak-* crash-* else # TEMP_CORPUS_DIR may contain many new inputs that only covers a small set of new branches. # Instead of copying all new inputs to the corpus directory, only copy back minimum number of new inputs that reach new branches. - ${BUILD_DIR_PATH}/bin/${TEST_NAME} -merge=1 "./corpus/${TEST_NAME}" "${TEMP_CORPUS_DIR}" \ - > ${TEST_NAME}_results.txt 2>&1 - - # Print number of new files and branches found in new Inputs (if any) - RESULTS=`grep -Eo "[0-9]+ new files .*$" ${TEST_NAME}_results.txt | tail -1` - printf ", ${RESULTS}\n" + ${S2N_ROOT}/build/bin/${TEST_NAME} \ + -merge=1 "./corpus/${TEST_NAME}" "${TEMP_CORPUS_DIR}" \ + > ${TEST_NAME}_results.txt 2>&1 if [ "$TESTS_PER_SEC" -lt $MIN_TEST_PER_SEC ]; then printf "\033[33;1mWARNING!\033[0m ${TEST_NAME} is only ${TESTS_PER_SEC} tests/sec, which is below ${MIN_TEST_PER_SEC}/sec! Fuzz tests are more effective at higher rates.\n\n" @@ -197,39 +117,12 @@ then printf "\033[31;1mERROR!\033[0m ${TEST_NAME} only covers ${FEATURE_COVERAGE} features, which is below ${MIN_FEATURES_COVERED}! This may be due to missing corpus files or a bug.\n" exit -1; fi - - # Store generated corpus files in the S3 bucket. - unset LD_PRELOAD - unset LD_LIBRARY_PATH - if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then - printf "Zipping corpus files...\n" - zip -r ./corpus/${TEST_NAME}.zip ./corpus/${TEST_NAME}/ - - printf "Uploading zipped corpus file to S3 bucket...\n" - aws s3 cp ./corpus/${TEST_NAME}.zip $CORPUS_UPLOAD_LOC/${TEST_NAME}/corpus.zip - fi fi - else cat ${TEST_NAME}_output.txt printf "\033[31;1mFAILED\033[0m %s, %6d features covered\n" "$TEST_INFO" $FEATURE_COVERAGE - - # Store corpus to S3 to be used for debugging if the test fails - unset LD_PRELOAD - unset LD_LIBRARY_PATH - if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then - printf "Zipping corpus files...\n" - zip -r ./corpus/${TEST_NAME}.zip ./corpus/${TEST_NAME}/ - - printf "Uploading zipped corpus file to S3 bucket...\n" - aws s3 cp ./corpus/${TEST_NAME}.zip $CORPUS_UPLOAD_LOC/${TEST_NAME}/corpus_$(date +%Y-%m-%d-%T).zip - fi - - # Store generated output files in the S3 bucket. - if [ "$ARTIFACT_UPLOAD_LOC" != "none" ]; then - printf "Uploading output files to S3 bucket...\n" - aws s3 cp ./${TEST_NAME}_output.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/output_$(date +%Y-%m-%d-%T).txt - aws s3 cp ./${TEST_NAME}_results.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/results_$(date +%Y-%m-%d-%T).txt - fi + # Store generated output files in the S3 bucket for debugging. + aws s3 cp ./tests/fuzz/${TEST_NAME}_output.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/output_$(date +%Y-%m-%d-%T).txt + aws s3 cp ./tests/fuzz/${TEST_NAME}_results.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/results_$(date +%Y-%m-%d-%T).txt exit -1 fi