apply tcpdump wrapper

nat-lab/bin/derp-server

@@ -2,6 +2,10 @@
 
 set -e
 
+# Backup IP tables
+iptables-save -f iptables_backup
+ip6tables-save -f ip6tables_backup
+
 /usr/bin/nordderper &
 
 sleep infinity

nat-lab/bin/dns-server.sh

@@ -1,2 +1,7 @@
 #!/bin/sh
+
+# Backup IP tables
+iptables-save -f iptables_backup
+ip6tables-save -f ip6tables_backup
+
 /opt/bin/dns-server 2>&1 | tee /dns-server.log

nat-lab/docker-compose.yml

@@ -423,6 +423,8 @@ services:
       - net.netfilter.nf_conntrack_tcp_timeout_fin_wait=15
     cap_drop:
       - ALL
+    cap_add:
+      - NET_ADMIN
     security_opt:
       - no-new-privileges:true
     networks:
@@ -549,6 +551,10 @@ services:
     hostname: dns-server-1
     image: nat-lab:base
     entrypoint: /opt/bin/dns-server.sh
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_ADMIN
     environment:
       PYTHONUNBUFFERED: 1
     networks:
@@ -561,6 +567,10 @@ services:
     hostname: dns-server-2
     image: nat-lab:base
     entrypoint: /opt/bin/dns-server.sh
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_ADMIN
     environment:
       PYTHONUNBUFFERED: 1
     networks:

nat-lab/tests/conftest.py

@@ -4,22 +4,21 @@
 import random
 import shutil
 import subprocess
+from contextlib import AsyncExitStack
 from datetime import datetime
 from helpers import SetupParameters
 from interderp_cli import InterDerpClient
 from itertools import combinations
-from mesh_api import start_tcpdump, stop_tcpdump
 from utils.bindings import TelioAdapterType
-from utils.connection import DockerConnection
 from utils.connection_util import (
     ConnectionTag,
-    container_id,
     DOCKER_GW_MAP,
     LAN_ADDR_MAP,
     new_connection_raw,
 )
 from utils.process import ProcessExecError
 from utils.router import IPStack
+from utils.tcpdump import make_tcpdump
 from utils.vm import windows_vm_util, mac_vm_util
 
 DERP_SERVER_1_ADDR = "http://10.0.10.1:8765"
@@ -31,6 +30,11 @@
 SETUP_CHECK_TIMEOUT_S = 30
 SETUP_CHECK_RETRIES = 5
 
+# pylint: disable=unnecessary-dunder-call
+TEST_SCOPE_ASYNC_EXIT_STACK = asyncio.run(AsyncExitStack().__aenter__())
+# pylint: disable=unnecessary-dunder-call
+SESSION_ASYNC_EXIT_STACK = asyncio.run(AsyncExitStack().__aenter__())
+
 
 def _cancel_all_tasks(loop: asyncio.AbstractEventLoop):
     to_cancel = asyncio.tasks.all_tasks(loop)
@@ -198,24 +202,25 @@ def pytest_make_parametrize_id(config, val):
 
 
 async def setup_check_interderp():
-    async with new_connection_raw(ConnectionTag.DOCKER_CONE_CLIENT_1) as connection:
-        if not isinstance(connection, DockerConnection):
-            raise Exception("Not docker connection")
-        containers = [
-            connection.container_name(),
-            "nat-lab-derp-01-1",
-            "nat-lab-derp-02-1",
-            "nat-lab-derp-03-1",
+    async with AsyncExitStack() as exit_stack:
+        connections = [
+            await exit_stack.enter_async_context(new_connection_raw(conn_tag))
+            for conn_tag in [
+                ConnectionTag.DOCKER_CONE_CLIENT_1,
+                ConnectionTag.DOCKER_DERP_1,
+                ConnectionTag.DOCKER_DERP_2,
+                ConnectionTag.DOCKER_DERP_3,
+            ]
         ]
-        start_tcpdump(containers)
-        try:
+
+        async with make_tcpdump(connections):
             for idx, (server1, server2) in enumerate(
                 combinations(
                     [DERP_SERVER_1_ADDR, DERP_SERVER_2_ADDR, DERP_SERVER_3_ADDR], 2
                 )
             ):
                 derp_test = InterDerpClient(
-                    connection,
+                    connections[0],
                     server1,
                     server2,
                     DERP_SERVER_1_SECRET_KEY,
@@ -224,8 +229,6 @@ async def setup_check_interderp():
                 )
                 await derp_test.execute()
                 await derp_test.save_logs()
-        finally:
-            stop_tcpdump(containers)
 
 
 SETUP_CHECKS = [
@@ -406,22 +409,55 @@ def pytest_runtest_setup():
 
 # pylint: disable=unused-argument
 def pytest_runtest_call(item):
-    start_tcpdump([f"nat-lab-dns-server-{i}-1" for i in range(1, 3)])
+    if os.environ.get("NATLAB_SAVE_LOGS") is None:
+        return
+
+    async def async_code():
+        connections = [
+            await TEST_SCOPE_ASYNC_EXIT_STACK.enter_async_context(
+                new_connection_raw(conn_tag)
+            )
+            for conn_tag in [
+                ConnectionTag.DOCKER_DNS_SERVER_1,
+                ConnectionTag.DOCKER_DNS_SERVER_2,
+            ]
+        ]
+        await TEST_SCOPE_ASYNC_EXIT_STACK.enter_async_context(make_tcpdump(connections))
+
+    asyncio.run(async_code())
 
 
 # pylint: disable=unused-argument
 def pytest_runtest_makereport(item, call):
+    if os.environ.get("NATLAB_SAVE_LOGS") is None:
+        return
+
+    async def async_code():
+        global TEST_SCOPE_ASYNC_EXIT_STACK
+        await TEST_SCOPE_ASYNC_EXIT_STACK.aclose()
+        # pylint: disable=unnecessary-dunder-call
+        TEST_SCOPE_ASYNC_EXIT_STACK = await AsyncExitStack().__aenter__()
+
     if call.when == "call":
-        stop_tcpdump([f"nat-lab-dns-server-{i}-1" for i in range(1, 3)])
+        asyncio.run(async_code())
 
 
 # pylint: disable=unused-argument
 def pytest_sessionstart(session):
     if os.environ.get("NATLAB_SAVE_LOGS") is None:
         return
 
+    async def async_code():
+        connections = [
+            await SESSION_ASYNC_EXIT_STACK.enter_async_context(
+                new_connection_raw(conn_tag)
+            )
+            for conn_tag in DOCKER_GW_MAP.values()
+        ]
+        await SESSION_ASYNC_EXIT_STACK.enter_async_context(make_tcpdump(connections))
+
     if not session.config.option.collectonly:
-        start_tcpdump({container_id(gw_tag) for gw_tag in DOCKER_GW_MAP.values()})
+        asyncio.run(async_code())
 
 
 # pylint: disable=unused-argument
@@ -430,10 +466,7 @@ def pytest_sessionfinish(session, exitstatus):
         return
 
     if not session.config.option.collectonly:
-        stop_tcpdump(
-            {container_id(gw_tag) for gw_tag in DOCKER_GW_MAP.values()}, "./logs"
-        )
-
+        asyncio.run(SESSION_ASYNC_EXIT_STACK.aclose())
         collect_nordderper_logs()
         collect_dns_server_logs()
         asyncio.run(collect_kernel_logs(session.items, "after_tests"))

nat-lab/tests/helpers.py

@@ -2,12 +2,11 @@
 import itertools
 import json
 import pytest
-from config import WG_SERVERS
 from contextlib import AsyncExitStack, asynccontextmanager
 from dataclasses import dataclass, field
 from datetime import datetime
 from itertools import product, zip_longest
-from mesh_api import Node, API, stop_tcpdump
+from mesh_api import Node, API
 from telio import Client
 from typing import AsyncIterator, List, Tuple, Optional, Union
 from utils.bindings import (
@@ -26,9 +25,11 @@
     ConnectionManager,
     ConnectionTag,
     new_connection_manager_by_tag,
+    new_connection_raw,
 )
 from utils.ping import ping
 from utils.router import IPStack
+from utils.tcpdump import make_tcpdump
 from uuid import UUID
 
 
@@ -294,6 +295,15 @@ async def setup_environment(
     )
 
     if prepare_vpn:
+        connections = [
+            await exit_stack.enter_async_context(new_connection_raw(conn_tag))
+            for conn_tag in [
+                ConnectionTag.DOCKER_NLX_1,
+                ConnectionTag.DOCKER_VPN_1,
+                ConnectionTag.DOCKER_VPN_2,
+            ]
+        ]
+        await exit_stack.enter_async_context(make_tcpdump(connections))
         api.prepare_all_vpn_servers()
 
     clients = await setup_clients(
@@ -317,18 +327,15 @@ async def setup_environment(
         ),
     )
 
-    try:
-        yield Environment(api, nodes, connection_managers, clients)
-
-        print(datetime.now(), "Checking connection limits")
-        for conn_manager in connection_managers:
-            if conn_manager.tracker:
-                violations = await conn_manager.tracker.find_conntracker_violations()
-                assert (
-                    violations is None
-                ), f"conntracker reported out of limits {violations}"
-    finally:
-        stop_tcpdump([server["container"] for server in WG_SERVERS])
+    yield Environment(api, nodes, connection_managers, clients)
+
+    print(datetime.now(), "Checking connection limits")
+    for conn_manager in connection_managers:
+        if conn_manager.tracker:
+            violations = await conn_manager.tracker.find_conntracker_violations()
+            assert (
+                violations is None
+            ), f"conntracker reported out of limits {violations}"
 
 
 async def setup_mesh_nodes(

nat-lab/tests/mesh_api.py

@@ -5,15 +5,12 @@
 import random
 import time
 import uuid
-from collections.abc import Iterable
-from concurrent.futures import ThreadPoolExecutor
 from config import DERP_SERVERS, LIBTELIO_IPV6_WG_SUBNET, WG_SERVERS
 from datetime import datetime
 from ipaddress import ip_address
 from typing import Dict, Any, List, Tuple, Optional
 from utils.bindings import Config, Server, Peer, PeerBase
 from utils.router import IPStack, IPProto, get_ip_address_type
-from utils.testing import get_current_test_log_path
 
 if platform.machine() != "x86_64":
     import pure_wg as Key
@@ -373,7 +370,6 @@ def generate_peer_config(node: Node, allowed_ips: str) -> str:
         )
 
         for node in node_list:
-            start_tcpdump([server_config["container"]])
             if "type" in server_config and server_config["type"] == "nordlynx":
                 priv_key = server_config["private_key"]
                 commands = [
@@ -445,51 +441,3 @@ def config_dynamic_nodes(
     def prepare_all_vpn_servers(self):
         for wg_server in WG_SERVERS:
             self.setup_vpn_servers(list(self.nodes.values()), wg_server)
-
-
-def start_tcpdump(container_names: Iterable[str]):
-    if os.environ.get("NATLAB_SAVE_LOGS") is None:
-        return
-
-    def aux(container_name):
-        # First make sure that no leftover processes/files will interfere
-        cmd = f"docker exec --privileged {container_name} killall -w tcpdump"
-        os.system(cmd)
-        cmd = f"docker exec --privileged {container_name} rm {PCAP_FILE_PATH}"
-        os.system(cmd)
-        cmd = f"docker exec -d --privileged {container_name} tcpdump -i any -U -w {PCAP_FILE_PATH}"
-        os.system(cmd)
-
-    with ThreadPoolExecutor() as executor:
-        executor.map(aux, container_names)
-
-
-def stop_tcpdump(container_names, store_in=None):
-    if os.environ.get("NATLAB_SAVE_LOGS") is None:
-        return
-    log_dir = get_current_test_log_path()
-    os.makedirs(log_dir, exist_ok=True)
-
-    def aux(container_name):
-        cmd = f"docker exec --privileged {container_name} killall -w tcpdump"
-        os.system(cmd)
-        path = find_unique_path_for_tcpdump(
-            store_in if store_in else log_dir, container_name
-        )
-        cmd = f"docker container cp {container_name}:{PCAP_FILE_PATH} {path}"
-        os.system(cmd)
-
-    with ThreadPoolExecutor() as executor:
-        executor.map(aux, container_names)
-
-
-def find_unique_path_for_tcpdump(log_dir, container_name):
-    candidate_path = f"{log_dir}/{container_name}.pcap"
-    counter = 1
-    # NOTE: counter starting from '1' means that the file will either have no suffix or
-    # will have a suffix starting from '2'. This is to make it clear that it's not the
-    # first log for that container/client.
-    while os.path.isfile(candidate_path):
-        counter += 1
-        candidate_path = f"./{log_dir}/{container_name}-{counter}.pcap"
-    return candidate_path