Skip to content

Commit

Permalink
Merge pull request #373 from Altinity/customizations/23.3.19-fips
Browse files Browse the repository at this point in the history 
23.3.19 FIPS Pre-release
  • Loading branch information
@Enmk
Enmk authored Mar 12, 2024
2 parents 19a57e6 + c5978a0 commit 0a8ac3b
Show file tree
Hide file tree
Showing 19 changed files with 338 additions and 54 deletions.
38 changes: 19 additions & 19 deletions .github/workflows/release_branches.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ jobs:
timeout_minutes: 180
runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
additional_envs: |
CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
CLICKHOUSE_STABLE_VERSION_SUFFIX=altinityfips
BuilderDebAarch64:
needs: [DockerHubPush]
Expand All @@ -152,7 +152,7 @@ jobs:
timeout_minutes: 180
runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
additional_envs: |
CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
CLICKHOUSE_STABLE_VERSION_SUFFIX=altinityfips
BuilderDebAsan:
needs: [DockerHubPush]
Expand All @@ -177,7 +177,7 @@ jobs:
runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
additional_envs: |
CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
BuilderDebTsan:
needs: [DockerHubPush]
uses: ./.github/workflows/reusable_build.yml
Expand Down Expand Up @@ -353,7 +353,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatelessTestTsan:
needs: [BuilderDebTsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -367,7 +367,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatelessTestUBsan:
needs: [BuilderDebUBsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -381,7 +381,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatelessTestMsan:
needs: [BuilderDebMsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -395,7 +395,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatelessTestDebug:
needs: [BuilderDebDebug]
uses: ./.github/workflows/reusable_test.yml
Expand Down Expand Up @@ -451,7 +451,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatefulTestTsan:
needs: [BuilderDebTsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -464,7 +464,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatefulTestMsan:
needs: [BuilderDebMsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -477,7 +477,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatefulTestUBsan:
needs: [BuilderDebUBsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -490,7 +490,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
FunctionalStatefulTestDebug:
needs: [BuilderDebDebug]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -503,7 +503,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
##############################################################################################
######################################### STRESS TESTS #######################################
##############################################################################################
Expand All @@ -517,7 +517,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 stress_check.py "$CHECK_NAME"
StressTestTsan:
needs: [BuilderDebTsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -528,7 +528,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 stress_check.py "$CHECK_NAME"
StressTestMsan:
needs: [BuilderDebMsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -539,7 +539,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 stress_check.py "$CHECK_NAME"
StressTestUBsan:
needs: [BuilderDebUBsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -550,7 +550,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 stress_check.py "$CHECK_NAME"
StressTestDebug:
needs: [BuilderDebDebug]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -577,7 +577,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
IntegrationTestsAnalyzerAsan:
needs: [BuilderDebAsan]
uses: ./.github/workflows/reusable_test.yml
Expand All @@ -590,7 +590,7 @@ jobs:
run_command: |
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
IntegrationTestsTsan:
needs: [BuilderDebTsan]
uses: ./.github/workflows/reusable_test.yml
Expand Down Expand Up @@ -627,7 +627,7 @@ jobs:
with:
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression
commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc
arch: release
arch: release
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }}

RegressionTestsAarch64:
Expand Down
4 changes: 4 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
"Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF)
but is not possible to satisfy" ON)

option(FIPS_CLICKHOUSE
"Build ClickHouse in FIPS mode: that is both BoringSSL and Poco are build in FIPS mode"
ON)

if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR)
else()
Expand Down
8 changes: 4 additions & 4 deletions cmake/autogenerated_versions.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ SET(VERSION_MINOR 3)
SET(VERSION_PATCH 19)
SET(VERSION_GITHASH 7228475d77afaf8a59d489694343593d3b650170)

SET(VERSION_TWEAK 33)
SET(VERSION_FLAVOUR altinitystable)
SET(VERSION_TWEAK 34)
SET(VERSION_FLAVOUR altinityfips)

SET(VERSION_DESCRIBE v23.3.19.33.altinitystable)
SET(VERSION_STRING 23.3.19.33.altinitystable)
SET(VERSION_DESCRIBE v23.3.19.34.altinityfips)
SET(VERSION_STRING 23.3.19.34.altinityfips)
# end of autochange
15 changes: 14 additions & 1 deletion cmake/split_debug_symbols.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ macro(clickhouse_split_debug_symbols)
message(FATAL_ERROR "Destination directory for stripped binary must be provided")
endif()

set(STRIP_EXTRA_ARGS "")
if (FIPS_CHLICKHOUSE)
# For FIPS tests (hash-break and to run properly, we need to keep some symbols
foreach(symbol_name IN ITEMS
BORINGSSL_bcm_rodata_start
BORINGSSL_bcm_rodata_end
BORINGSSL_bcm_text_start
BORINGSSL_bcm_text_end
)
set(STRIP_EXTRA_ARGS "${STRIP_EXTRA_ARGS} -K ${symbol_name}")
endforeach()
endif()

add_custom_command(TARGET ${STRIP_TARGET} POST_BUILD
COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/lib/debug/bin"
COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/bin"
Expand All @@ -23,7 +36,7 @@ macro(clickhouse_split_debug_symbols)
COMMAND "${OBJCOPY_PATH}" --only-keep-debug "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
COMMAND chmod 0644 "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
# Strips binary, sections '.note' & '.comment' are removed in line with Debian's stripping policy: www.debian.org/doc/debian-policy/ch-files.html, section '.clickhouse.hash' is needed for integrity check:
COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash ${STRIP_EXTRA_ARGS} "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
# Associate stripped binary with debug symbols:
COMMAND "${OBJCOPY_PATH}" --add-gnu-debuglink "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
COMMENT "Stripping clickhouse binary" VERBATIM
Expand Down
120 changes: 120 additions & 0 deletions contrib/boringssl-cmake/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,124 @@ if(NOT ENABLE_SSL)
return()
endif()

if(FIPS_CLICKHOUSE)

set(BORINGSSL_BUILD_DIR "${CMAKE_BINARY_DIR}/go1.19-boringssl-build")
set(BORINGSSL_BINARIES_DIR "${BORINGSSL_BUILD_DIR}/output")

message("Will build BoringSSL in FIPS mode according to go1.19 recipe...")
# build BoringSSL in FIPS mode accoring to the Security Policy:
# https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp3678.pdf
# We re-use golang-1.19 routine, for that we need only 3 files from golang repo, so no need to checkout a full repository:
# - src/crypto/internal/boring/Dockerfile - build environment
# - src/crypto/internal/boring/build.sh - build and test script
# - src/crypto/internal/boring/goboringcrypto.h - required for producing golangs syso (whuch we do not need), but build will fail without it.
# hashsums were calculated on 28 Dec 2022

message("Downloading build files from go1.19 github...")
file(DOWNLOAD
https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/goboringcrypto.h
${BORINGSSL_BUILD_DIR}/goboringcrypto.h
EXPECTED_HASH SHA256=2bea41082e0cc2bdfc6d5fccc64544cb52cc889e6e99330a6b423f04fef48a57
SHOW_PROGRESS
)

file(DOWNLOAD
https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/Dockerfile
${BORINGSSL_BUILD_DIR}/Dockerfile
EXPECTED_HASH SHA256=c7d3d13d028f542af5dd9f173ad7b7a29bc398f61bb5dd228e6da48b05ea5487
SHOW_PROGRESS
)

file(DOWNLOAD
https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/build.sh
${BORINGSSL_BUILD_DIR}/build.sh
EXPECTED_HASH SHA256=b4daa2ee2c1ce735c8720eb22d4ef21f9a7a03c737230bddf3582b0fed1b3728
SHOW_PROGRESS
)

# Build driver - the script that triggers the build and pulls out results from docker container
file(WRITE ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh
"#!/bin/bash
set -ex
OUTPUT_DIR=$1
shift
docker build . -t boringssl-builder
readonly id=$(docker create boringssl-builder)
docker start -a $id #| tr -dc \\\\x0-\\\\x9
mkdir -p $OUTPUT_DIR
# Copy build artifacts
docker cp $id:/boring/boringssl/build/ssl/libssl.a $OUTPUT_DIR
docker cp $id:/boring/boringssl/build/crypto/libcrypto.a $OUTPUT_DIR
docker cp $id:/boring/boringssl/build/decrepit/libdecrepit.a $OUTPUT_DIR
docker cp $id:/boring/boringssl/include $OUTPUT_DIR
docker rm $id"
)

# patch a single file in krb5 that relies on file missing from this version of BoringSSL
SET(krb5_filb_to_patch ${PROJECT_SOURCE_DIR}/contrib/krb5/src/lib/crypto/openssl/enc_provider/aes.c)
message("Patching ${krb5_filb_to_patch} to allow building against older version of BoringSSL")
file(READ ${krb5_filb_to_patch} FILE_CONTENTS)
string(REPLACE "#include <openssl/modes.h>" "//#include <openssl/modes.h>" FILE_CONTENTS "${FILE_CONTENTS}")
file(WRITE ${krb5_filb_to_patch} "${FILE_CONTENTS}")

message("Creating directory for BoringSSL binaries and includes in ${BORINGSSL_BINARIES_DIR}")
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory "${BORINGSSL_BINARIES_DIR}/include")

add_custom_target(build-boringssl
# COMMENT "Build BoringSSL in FIPS mode with docker (using go1.19 build suite)"
DEPENDS ${BORINGSSL_BINARIES_DIR}/libssl.a ${BORINGSSL_BINARIES_DIR}/libcrypto.a
)

add_custom_command(
OUTPUT
"${BORINGSSL_BUILD_DIR}/output/libssl.a"
"${BORINGSSL_BUILD_DIR}/output/libcrypto.a"
"${BORINGSSL_BUILD_DIR}/output/libdecrepit.a"
COMMENT "Building BoringSSL in FIPS mode using Docker"
COMMAND bash -c "chmod +x ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BUILD_DIR}/build.sh"
COMMAND bash -c "${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BINARIES_DIR}"
WORKING_DIRECTORY ${BORINGSSL_BUILD_DIR}
USES_TERMINAL # To stream output
DEPENDS
${BORINGSSL_BUILD_DIR}/build.sh
${BORINGSSL_BUILD_DIR}/goboringcrypto.h
${BORINGSSL_BUILD_DIR}/Dockerfile
)

add_library(_crypto UNKNOWN IMPORTED GLOBAL)
add_dependencies(_crypto build-boringssl)
set_target_properties(_crypto PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libcrypto.a"
INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
)

add_library(_decrepit UNKNOWN IMPORTED)
add_dependencies(_decrepit build-boringssl)
set_target_properties(_decrepit PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libdecrepit.a"
INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
)

add_library(_ssl UNKNOWN IMPORTED GLOBAL)
add_dependencies(_ssl _crypto)
set_target_properties(_ssl PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libssl.a"
INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
INTERFACE_LINK_LIBRARIES _decrepit
)

else() # FIPS_CLICKHOUSE

# Copyright (c) 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
Expand Down Expand Up @@ -687,6 +805,8 @@ target_include_directories(_ssl SYSTEM PUBLIC "${BORINGSSL_SOURCE_DIR}/include")

target_compile_options(_crypto PRIVATE -Wno-gnu-anonymous-struct)

endif() # FIPS_CLICKHOUSE

add_library(OpenSSL::Crypto ALIAS _crypto)
add_library(OpenSSL::SSL ALIAS _ssl)

Expand Down
Loading

0 comments on commit 0a8ac3b

Please sign in to comment.