Enable keyfromlabel and enginepkcs11 systemtests with pkcs11-provider

- Simplify configuration management by deducing SoftHSM module path
  from openssl config

- Determine the engine flag (-E) value from openssl config

- Drop unused/unneeded environment variables

- Run pkcs11-provider tests on Debian "sid" ossl3 flavor

Author: fabled

.gitlab-ci.yml

@@ -864,16 +864,23 @@ scan-build:
     when: on_failure
 
 # Jobs for strict OpenSSL 3.x (no deprecated) GCC builds on Debian "sid" (amd64)
+# Run with pkcs11-provider tests
 
 gcc:ossl3:sid:amd64:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -O3 -DOPENSSL_NO_DEPRECATED=1 -DOPENSSL_API_COMPAT=30000"
+    # See https://gitlab.isc.org/isc-projects/bind9/-/issues/3444
+    EXTRA_CONFIGURE: "--without-jemalloc --disable-leak-detection"
     RUN_MAKE_INSTALL: 1
   <<: *debian_sid_amd64_image
   <<: *build_job
 
 system:gcc:ossl3:sid:amd64:
+  # Set up environment variables to run pkcs11-provider system tests
+  variables:
+    OPENSSL_CONF: "/var/tmp/etc/openssl-provider.cnf"
+    SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf"
   <<: *debian_sid_amd64_image
   <<: *system_test_job
   needs:
@@ -1207,10 +1214,8 @@ gcc:bullseye:amd64:
 system:gcc:bullseye:amd64:
   # Set up environment variables that allow the "keyfromlabel" system test to be run
   variables:
-    DEFAULT_OPENSSL_CONF: "/etc/ssl/openssl.cnf"
     OPENSSL_CONF: "/var/tmp/etc/openssl.cnf"
     SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf"
-    SOFTHSM2_MODULE: "/usr/lib/softhsm/libsofthsm2.so"
   <<: *debian_bullseye_amd64_image
   <<: *system_test_job
   needs:
@@ -1240,10 +1245,8 @@ clang:bookworm:amd64:
 system:clang:bookworm:amd64:
   # Set up environment variables that allow the "keyfromlabel" system test to be run
   variables:
-    DEFAULT_OPENSSL_CONF: "/etc/ssl/openssl.cnf"
     OPENSSL_CONF: "/var/tmp/etc/openssl.cnf"
     SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf"
-    SOFTHSM2_MODULE: "/usr/lib/softhsm/libsofthsm2.so"
   <<: *debian_bookworm_amd64_image
   <<: *system_test_job
   needs:

bin/tests/system/conf.sh.common

@@ -728,3 +728,27 @@ copy_setports() {
         -e "s/@DISABLED_BITS@/${DISABLED_BITS}/g" \
         $1 > $2
 }
+
+# parse_openssl_config - Parse OpenSSL configuration for HSM settings
+#
+# Will set SOFTHSM2_MODULE, OPENSSL_ENGINE and ENGINE_ARG based on openssl configuration.
+parse_openssl_config() {
+    ENGINE_ARG=""
+    [ -f "$OPENSSL_CONF" ] || return 0
+    while IFS="=" read key val; do
+        # trim variables
+        key="${key## }"
+        key="${key%% }"
+        val="${val## }"
+        val="${val%% }"
+        case "$key" in
+        "engine_id")
+            OPENSSL_ENGINE="$val"
+            ENGINE_ARG="-E $OPENSSL_ENGINE"
+            ;;
+        "MODULE_PATH"|"pkcs11-module-path")
+            SOFTHSM2_MODULE="$val"
+            ;;
+        esac
+    done < "$OPENSSL_CONF"
+}

bin/tests/system/enginepkcs11/clean.sh

@@ -26,10 +26,10 @@ rm -f ns1/*.example.db ns1/*.example.db.signed
 rm -f ns1/*.kskid1 ns1/*.kskid2 ns1/*.zskid1 ns1/*.zskid2
 rm -f ns1/dig.out.*
 rm -f ns1/K*
-rm -f ns1/named.conf ns1/named.run ns1/named.memstats
+rm -f ns1/named.conf ns1/named.args ns1/named.run ns1/named.memstats
 rm -f ns1/update.cmd.*
 rm -f ns1/update.log.*
 rm -f ns1/verify.out.*
 rm -f ns1/zone.*.signed.jnl ns1/zone.*.signed.jbk
 
-softhsm2-util --delete-token --token "softhsm2-enginepkcs11" >/dev/null 2>&1 || echo_i "softhsm2-enginepkcs11 token not found for cleaning"
+OPENSSL_CONF= softhsm2-util --delete-token --token "softhsm2-enginepkcs11" >/dev/null 2>&1 || echo_i "softhsm2-enginepkcs11 token not found for cleaning"

bin/tests/system/enginepkcs11/ns1/named.args

@@ -0,0 +1 @@
+@ENGINE_ARGS@ -D enginepkcs11-ns1 -X named.lock -m record -c named.conf -d 99 -U 4 -T maxcachesize=2097152

bin/tests/system/enginepkcs11/ns1/named.args.in

@@ -13,9 +13,20 @@
 
 . ../conf.sh
 
-if [ -n "${SOFTHSM2_MODULE}" ] && command -v softhsm2-util >/dev/null; then
-	exit 0
-fi
+[ -n "${SOFTHSM2_CONF}" ] || {
+    echo_i "skip: softhsm2 configuration not available"
+    exit 255
+}
 
-echo_i "skip: softhsm2-util not available"
-exit 255
+parse_openssl_config
+[ -f "$SOFTHSM2_MODULE" ] || {
+    echo_i "skip: softhsm2 module not available"
+    exit 1
+}
+
+for _bin in softhsm2-util pkcs11-tool; do
+    command -v "$_bin" >/dev/null || {
+        echo_i "skip: $_bin not available"
+        exit 1
+    }
+done

bin/tests/system/enginepkcs11/prereq.sh

@@ -16,12 +16,14 @@
 
 set -e
 
-softhsm2-util --init-token --free --pin 1234 --so-pin 1234 --label "softhsm2-enginepkcs11" | awk '/^The token has been initialized and is reassigned to slot/ { print $NF }'
+OPENSSL_CONF= softhsm2-util --init-token --free --pin 1234 --so-pin 1234 --label "softhsm2-enginepkcs11" | awk '/^The token has been initialized and is reassigned to slot/ { print $NF }'
 
 printf '%s' "${HSMPIN:-1234}" > pin
+parse_openssl_config
 PWD=$(pwd)
 
 copy_setports ns1/named.conf.in ns1/named.conf
+sed -e "s/@ENGINE_ARGS@/${ENGINE_ARG}/g" < ns1/named.args.in > ns1/named.args
 
 keygen() {
 	type="$1"
@@ -31,7 +33,7 @@ keygen() {
 
 	label="${id}-${zone}"
 	p11id=$(echo "${label}" | openssl sha1 -r | awk '{print $1}')
-	pkcs11-tool --module $SOFTHSM2_MODULE --token-label "softhsm2-enginepkcs11" -l -k --key-type $type:$bits --label "${label}" --id "${p11id}" --pin $(cat $PWD/pin) > pkcs11-tool.out.$zone.$id 2> pkcs11-tool.err.$zone.$id || return 1
+	OPENSSL_CONF= pkcs11-tool --module $SOFTHSM2_MODULE --token-label "softhsm2-enginepkcs11" -l -k --key-type $type:$bits --label "${label}" --id "${p11id}" --pin $(cat $PWD/pin) > pkcs11-tool.out.$zone.$id 2> pkcs11-tool.err.$zone.$id || return 1
 }
 
 keyfromlabel() {
@@ -41,7 +43,7 @@ keyfromlabel() {
 	dir="$4"
         shift 4
 
-	$KEYFRLAB -K $dir -E pkcs11 -a $alg -l "token=softhsm2-enginepkcs11;object=${id}-${zone};pin-source=$PWD/pin" "$@" $zone >> keyfromlabel.out.$zone.$id 2> keyfromlabel.err.$zone.$id || return 1
+	$KEYFRLAB $ENGINE_ARG -K $dir -a $alg -l "pkcs11:token=softhsm2-enginepkcs11;object=${id}-${zone};pin-source=$PWD/pin" "$@" $zone >> keyfromlabel.out.$zone.$id 2> keyfromlabel.err.$zone.$id || return 1
 	cat keyfromlabel.out.$zone.$id
 }
 
@@ -86,7 +88,7 @@ do
 
 		echo_i "Sign zone with $ksk1 $zsk1"
 		cat "$infile" "${dir}/${ksk1}.key" "${dir}/${zsk1}.key" > "${dir}/${zonefile}"
-		$SIGNER -K $dir -E pkcs11 -S -a -g -O full -o "$zone" "${dir}/${zonefile}" > signer.out.$zone || ret=1
+		$SIGNER $ENGINE_ARG -K $dir -S -a -g -O full -o "$zone" "${dir}/${zonefile}" > signer.out.$zone || ret=1
 		test "$ret" -eq 0 || exit 1
 
 		echo_i "Generate successor keys $alg $type:$bits for zone $zone"

bin/tests/system/enginepkcs11/setup.sh

@@ -16,6 +16,7 @@ set -e
 # shellcheck source=conf.sh
 . ../conf.sh
 
+parse_openssl_config
 PWD=$(pwd)
 
 status=0

bin/tests/system/enginepkcs11/tests.sh

@@ -24,4 +24,4 @@ rm -f keyfromlabel.out.*
 rm -f pkcs11-tool.out.*
 rm -f signer.out.*
 
-softhsm2-util --delete-token --token "softhsm2-keyfromlabel" >/dev/null 2>&1 || echo_i "softhsm2-keyfromlabel token not found for cleaning"
+OPENSSL_CONF= softhsm2-util --delete-token --token "softhsm2-keyfromlabel" >/dev/null 2>&1 || echo_i "softhsm2-keyfromlabel token not found for cleaning"

bin/tests/system/keyfromlabel/clean.sh

@@ -13,9 +13,20 @@
 
 . ../conf.sh
 
-if [ -n "${SOFTHSM2_MODULE}" ] && command -v pkcs11-tool >/dev/null && command -v softhsm2-util >/dev/null; then
-	exit 0
-fi
+[ -n "${SOFTHSM2_CONF}" ] || {
+    echo_i "skip: softhsm2 configuration not available"
+    exit 255
+}
 
-echo_i "skip: softhsm2-util or pkcs11-tool not available"
-exit 255
+parse_openssl_config
+[ -f "$SOFTHSM2_MODULE" ] || {
+    echo_i "skip: softhsm2 module not available"
+    exit 1
+}
+
+for _bin in softhsm2-util pkcs11-tool; do
+    command -v "$_bin" >/dev/null || {
+        echo_i "skip: $_bin not available"
+        exit 1
+    }
+done