feature(client-encrypt): enable peer verification for stress commands

Peer verification is now enabled by default for cassandra-stress, scylla-bench,
and latte stress tools when client encryption is configured in Scylla. This ensures
enhanced security by verifying if peer certificate is signed by the trusted CA and
that the hostname/IP of the peer matches SAN specified in the peer's certificate.

Closes: scylladb/qa-tasks#1728

artifacts_test.py

@@ -128,7 +128,8 @@ def check_cluster_name(self):
     def run_cassandra_stress(self, args: str):
         stress_cmd = f"{self.node.add_install_prefix(STRESS_CMD)} {args} -node {self.node.ip_address}"
         if self.params.get('client_encrypt'):
-            transport_str = c_s_transport_str(self.params.get('client_encrypt_mtls'))
+            transport_str = c_s_transport_str(
+                self.params.get('peer_verification'), self.params.get('client_encrypt_mtls'))
             stress_cmd += f" -transport '{transport_str}'"
 
         result = self.node.remoter.run(stress_cmd)

defaults/test_default.yaml

@@ -76,6 +76,7 @@ parallel_node_operations: false  # supported from Scylla 6.0
 
 server_encrypt: false
 client_encrypt: false
+peer_verification: true  # when client encryption is used, peer verification is enabled by default
 client_encrypt_mtls: false
 server_encrypt_mtls: false
 

docs/configuration_options.md

@@ -762,6 +762,15 @@ when enable scylla will use encryption on the client side
 **type:** boolean
 
 
+## **peer_verification** / SCT_PEER_VERIFICATION
+
+enable peer verification for encrypted communication
+
+**default:** True
+
+**type:** boolean
+
+
 ## **client_encrypt_mtls** / SCT_CLIENT_ENCRYPT_MTLS
 
 when enabled scylla will enforce mutual authentication when client-to-node encryption is enabled
@@ -3523,3 +3532,12 @@ Error thresholds for latency decorator. Defined by dict: {<write, read, mixed>:
 **default:** {'write': {'default': {'P90 write': {'fixed_limit': 5}, 'P99 write': {'fixed_limit': 10}}}, 'read': {'default': {'P90 read': {'fixed_limit': 5}, 'P99 read': {'fixed_limit': 10}}}, 'mixed': {'default': {'P90 write': {'fixed_limit': 5}, 'P90 read': {'fixed_limit': 5}, 'P99 write': {'fixed_limit': 10}, 'P99 read': {'fixed_limit': 10}}}}
 
 **type:** dict_or_str
+
+
+## **workload_name** / SCT_WORKLOAD_NAME
+
+Workload name, can be: write|read|mixed|unset.Used for e.g. latency_calculator_decorator (use with 'use_hdr_cs_histogram' set to true).If unset, workload is taken from test name.
+
+**default:** N/A
+
+**type:** str (appendable)

sdcm/provision/helpers/certificate.py

@@ -327,12 +327,13 @@ def update_certificate(node: BaseNode) -> None:
         crt_file.write(new_cert.public_bytes(serialization.Encoding.PEM))
 
 
-def c_s_transport_str(client_mtls: bool) -> str:
+def c_s_transport_str(peer_verification: bool, client_mtls: bool) -> str:
     """Build transport string for cassandra-stress."""
     transport_str = f'truststore={SCYLLA_SSL_CONF_DIR}/{TLSAssets.JKS_TRUSTSTORE} truststore-password=cassandra'
+    if peer_verification:
+        transport_str += ' hostname-verification=true'
     if client_mtls:
-        transport_str = (
-            f'{transport_str} keystore={SCYLLA_SSL_CONF_DIR}/{TLSAssets.PKCS12_KEYSTORE} keystore-password=cassandra')
+        transport_str += f' keystore={SCYLLA_SSL_CONF_DIR}/{TLSAssets.PKCS12_KEYSTORE} keystore-password=cassandra'
     return transport_str
 
 

sdcm/sct_config.py

@@ -571,6 +571,9 @@ class SCTConfiguration(dict):
         dict(name="client_encrypt", env="SCT_CLIENT_ENCRYPT", type=boolean,
              help="when enable scylla will use encryption on the client side"),
 
+        dict(name="peer_verification", env="SCT_PEER_VERIFICATION", type=boolean,
+             help="enable peer verification for encrypted communication"),
+
         dict(name="client_encrypt_mtls", env="SCT_CLIENT_ENCRYPT_MTLS", type=boolean,
              help="when enabled scylla will enforce mutual authentication when client-to-node encryption is enabled"),
 

sdcm/scylla_bench_thread.py

@@ -174,15 +174,13 @@ def create_stress_cmd(self, stress_cmd, loader, cmd_runner):
                                               verbose=True)
                 stress_cmd = f'{stress_cmd.strip()} -tls -tls-ca-cert-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CA_CERT}'
 
+                if self.params.get("peer_verification"):
+                    stress_cmd = f'{stress_cmd.strip()} -tls-host-verification'
                 if self.params.get("client_encrypt_mtls"):
                     stress_cmd = (
                         f'{stress_cmd.strip()} -tls-client-key-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_KEY} '
                         f'-tls-client-cert-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_CERT}')
 
-                    # TBD: update after https://github.com/scylladb/scylla-bench/issues/140 is resolved
-                    # server_names = ' '.join(f'-tls-server-name {ip}' for ip in ips.split(","))
-                    # stress_cmd = f'{stress_cmd.strip()} -tls-host-verification {server_names}'
-
         return stress_cmd
 
     def _run_stress(self, loader, loader_idx, cpu_idx):  # pylint: disable=too-many-locals

sdcm/stress/latte_thread.py

@@ -122,6 +122,9 @@ def build_stress_cmd(self, cmd_runner, loader):  # pylint: disable=too-many-loca
                            f'--ssl-cert {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_CERT} '
                            f'--ssl-key {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_KEY}')
 
+            if self.params['peer_verification']:
+                ssl_config += ' --ssl-peer-verification'
+
         auth_config = ''
         if credentials := self.loader_set.get_db_auth():
             auth_config = f' --user {credentials[0]} --password {credentials[1]}'

sdcm/stress_thread.py

@@ -205,7 +205,8 @@ def create_stress_cmd(self, cmd_runner, keyspace_idx, loader):  # pylint: disabl
             # put the credentials into the right place into -mode section
             stress_cmd = re.sub(r'(-mode.*?)-', r'\1 user={} password={} -'.format(*credentials), stress_cmd)
         if self.client_encrypt and 'transport' not in stress_cmd:
-            transport_str = c_s_transport_str(self.params.get('client_encrypt_mtls'))
+            transport_str = c_s_transport_str(
+                self.params.get('peer_verification'), self.params.get('client_encrypt_mtls'))
             stress_cmd += f" -transport '{transport_str}'"
 
         stress_cmd = self.adjust_cmd_connection_options(stress_cmd, loader, cmd_runner)

unit_tests/conftest.py

@@ -73,26 +73,28 @@ def fixture_docker_scylla(request: pytest.FixtureRequest, params):  # pylint: di
     docker_version = docker_scylla_args.get('image', "scylladb/scylla-nightly:6.1.0-dev-0.20240605.2c3f7c996f98")
     cluster = LocalScyllaClusterDummy(params=params)
 
+    ssl_dir = (Path(__file__).parent.parent / 'data_dir' / 'ssl_conf').absolute()
+    extra_docker_opts = (f'-p {ALTERNATOR_PORT} -p {BaseNode.CQL_PORT} --cpus="1" -v {entryfile_path}:/entry.sh'
+                         f' -v {ssl_dir}:{SCYLLA_SSL_CONF_DIR}'
+                         ' --entrypoint /entry.sh')
+
+    scylla = RemoteDocker(LocalNode("scylla", cluster), image_name=docker_version,
+                          command_line=f"--smp 1 {alternator_flags}",
+                          extra_docker_opts=extra_docker_opts, docker_network=docker_network)
+
     if ssl:
         curr_dir = os.getcwd()
         try:
             os.chdir(Path(__file__).parent.parent)
             localhost = LocalHost(user_prefix='unit_test_fake_user', test_id='unit_test_fake_test_id')
             create_ca(localhost)
             create_certificate(CLIENT_FACING_CERTFILE, CLIENT_FACING_KEYFILE, cname="scylladb",
-                               ca_cert_file=CA_CERT_FILE, ca_key_file=CA_KEY_FILE)
+                               ca_cert_file=CA_CERT_FILE, ca_key_file=CA_KEY_FILE, ip_addresses=[scylla.ip_address])
             create_certificate(CLIENT_CERT_FILE, CLIENT_KEY_FILE, cname="scylladb",
                                ca_cert_file=CA_CERT_FILE, ca_key_file=CA_KEY_FILE)
         finally:
             os.chdir(curr_dir)
-    ssl_dir = (Path(__file__).parent.parent / 'data_dir' / 'ssl_conf').absolute()
-    extra_docker_opts = (f'-p {ALTERNATOR_PORT} -p {BaseNode.CQL_PORT} --cpus="1" -v {entryfile_path}:/entry.sh'
-                         f' -v {ssl_dir}:{SCYLLA_SSL_CONF_DIR}'
-                         ' --entrypoint /entry.sh')
 
-    scylla = RemoteDocker(LocalNode("scylla", cluster), image_name=docker_version,
-                          command_line=f"--smp 1 {alternator_flags}",
-                          extra_docker_opts=extra_docker_opts, docker_network=docker_network)
     cluster.nodes = [scylla]
     DummyRemoter = collections.namedtuple('DummyRemoter', ['run', 'sudo'])
     scylla.remoter = DummyRemoter(run=scylla.run, sudo=scylla.run)