From fe50f1a9292b34e168b35453f2cfc2aee2ca4843 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 2 Jul 2024 05:22:59 +0000 Subject: [PATCH 01/64] T6486: T6379: Rewrite generate openvpn client-config This command helps to generate users `.ovpn` files Rewrite `generate openvpn client-config` to use Config() It needs to get the default values as `ConfigTreeQuery` is not supporting default values. Fixed "ignores configured protocol type" if TCP is used Fixed lzo, was used even if lzo not configured Fixed encryption is not parse the dict --- src/op_mode/generate_ovpn_client_file.py | 113 +++++++++++++---------- 1 file changed, 64 insertions(+), 49 deletions(-) diff --git a/src/op_mode/generate_ovpn_client_file.py b/src/op_mode/generate_ovpn_client_file.py index 2d96fe217df..974f7d9b68c 100755 --- a/src/op_mode/generate_ovpn_client_file.py +++ b/src/op_mode/generate_ovpn_client_file.py @@ -19,42 +19,53 @@ from jinja2 import Template from textwrap import fill -from vyos.configquery import ConfigTreeQuery +from vyos.config import Config from vyos.ifconfig import Section client_config = """ client nobind -remote {{ remote_host }} {{ port }} +remote {{ local_host if local_host else 'x.x.x.x' }} {{ port }} remote-cert-tls server -proto {{ 'tcp-client' if protocol == 'tcp-active' else 'udp' }} -dev {{ device }} -dev-type {{ device }} +proto {{ 'tcp-client' if protocol == 'tcp-passive' else 'udp' }} +dev {{ device_type }} +dev-type {{ device_type }} persist-key persist-tun verb 3 # Encryption options +{# Define the encryption map #} +{% set encryption_map = { + 'des': 'DES-CBC', + '3des': 'DES-EDE3-CBC', + 'bf128': 'BF-CBC', + 'bf256': 'BF-CBC', + 'aes128gcm': 'AES-128-GCM', + 'aes128': 'AES-128-CBC', + 'aes192gcm': 'AES-192-GCM', + 'aes192': 'AES-192-CBC', + 'aes256gcm': 'AES-256-GCM', + 'aes256': 'AES-256-CBC' +} %} + {% if encryption is defined and encryption is not none %} -{% if encryption.cipher is defined and encryption.cipher is not none %} -cipher {{ encryption.cipher }} -{% if encryption.cipher == 'bf128' %} -keysize 128 -{% elif encryption.cipher == 'bf256' %} -keysize 256 +{% if encryption.ncp_ciphers is defined and encryption.ncp_ciphers is not none %} +cipher {% for algo in encryption.ncp_ciphers %} +{{ encryption_map[algo] if algo in encryption_map.keys() else algo }}{% if not loop.last %}:{% endif %} +{% endfor %} + +data-ciphers {% for algo in encryption.ncp_ciphers %} +{{ encryption_map[algo] if algo in encryption_map.keys() else algo }}{% if not loop.last %}:{% endif %} +{% endfor %} {% endif %} -{% endif %} -{% if encryption.ncp_ciphers is defined and encryption.ncp_ciphers is not none %} -data-ciphers {{ encryption.ncp_ciphers }} -{% endif %} {% endif %} {% if hash is defined and hash is not none %} auth {{ hash }} {% endif %} -keysize 256 -comp-lzo {{ '' if use_lzo_compression is defined else 'no' }} +{{ 'comp-lzo' if use_lzo_compression is defined else '' }} -----BEGIN CERTIFICATE----- @@ -79,7 +90,7 @@ """ -config = ConfigTreeQuery() +config = Config() base = ['interfaces', 'openvpn'] if not config.exists(base): @@ -89,10 +100,22 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("-i", "--interface", type=str, help='OpenVPN interface the client is connecting to', required=True) - parser.add_argument("-a", "--ca", type=str, help='OpenVPN CA cerificate', required=True) - parser.add_argument("-c", "--cert", type=str, help='OpenVPN client cerificate', required=True) - parser.add_argument("-k", "--key", type=str, help='OpenVPN client cerificate key', action="store") + parser.add_argument( + "-i", + "--interface", + type=str, + help='OpenVPN interface the client is connecting to', + required=True, + ) + parser.add_argument( + "-a", "--ca", type=str, help='OpenVPN CA cerificate', required=True + ) + parser.add_argument( + "-c", "--cert", type=str, help='OpenVPN client cerificate', required=True + ) + parser.add_argument( + "-k", "--key", type=str, help='OpenVPN client cerificate key', action="store" + ) args = parser.parse_args() interface = args.interface @@ -114,33 +137,25 @@ if not config.exists(['pki', 'certificate', cert, 'private', 'key']): exit(f'OpenVPN certificate key "{key}" does not exist!') - ca = config.value(['pki', 'ca', ca, 'certificate']) + config = config.get_config_dict( + base + [interface], + key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True, + with_pki=True, + ) + + ca = config['pki']['ca'][ca]['certificate'] ca = fill(ca, width=64) - cert = config.value(['pki', 'certificate', cert, 'certificate']) + cert = config['pki']['certificate'][cert]['certificate'] cert = fill(cert, width=64) - key = config.value(['pki', 'certificate', key, 'private', 'key']) + key = config['pki']['certificate'][key]['private']['key'] key = fill(key, width=64) - remote_host = config.value(base + [interface, 'local-host']) - - ovpn_conf = config.get_config_dict(base + [interface], key_mangling=('-', '_'), get_first_key=True) - - port = '1194' if 'local_port' not in ovpn_conf else ovpn_conf['local_port'] - proto = 'udp' if 'protocol' not in ovpn_conf else ovpn_conf['protocol'] - device = 'tun' if 'device_type' not in ovpn_conf else ovpn_conf['device_type'] - - config = { - 'interface' : interface, - 'ca' : ca, - 'cert' : cert, - 'key' : key, - 'device' : device, - 'port' : port, - 'proto' : proto, - 'remote_host' : remote_host, - 'address' : [], - } - -# Clear out terminal first -print('\x1b[2J\x1b[H') -client = Template(client_config, trim_blocks=True).render(config) -print(client) + + config['ca'] = ca + config['cert'] = cert + config['key'] = key + config['port'] = '1194' if 'local_port' not in config else config['local_port'] + + client = Template(client_config, trim_blocks=True).render(config) + print(client) From c509d0e6caae55106a2fbde3059652a493ed3903 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Mon, 8 Jul 2024 16:38:22 +0600 Subject: [PATCH 02/64] T6362: Create conntrack logger daemon --- data/templates/conntrack/sysctl.conf.j2 | 3 +- debian/control | 1 + .../include/conntrack/log-common.xml.i | 20 - .../include/conntrack/log-protocols.xml.i | 26 + interface-definitions/system_conntrack.xml.in | 81 +++- .../scripts/cli/test_system_conntrack.py | 35 +- src/conf_mode/system_conntrack.py | 21 +- src/services/vyos-conntrack-logger | 458 ++++++++++++++++++ src/systemd/vyos-conntrack-logger.service | 21 + 9 files changed, 620 insertions(+), 46 deletions(-) delete mode 100644 interface-definitions/include/conntrack/log-common.xml.i create mode 100644 interface-definitions/include/conntrack/log-protocols.xml.i create mode 100755 src/services/vyos-conntrack-logger create mode 100644 src/systemd/vyos-conntrack-logger.service diff --git a/data/templates/conntrack/sysctl.conf.j2 b/data/templates/conntrack/sysctl.conf.j2 index 554512f4d10..cd6c34edeca 100644 --- a/data/templates/conntrack/sysctl.conf.j2 +++ b/data/templates/conntrack/sysctl.conf.j2 @@ -6,4 +6,5 @@ net.netfilter.nf_conntrack_max = {{ table_size }} net.ipv4.tcp_max_syn_backlog = {{ tcp.half_open_connections }} net.netfilter.nf_conntrack_tcp_loose = {{ '1' if tcp.loose is vyos_defined('enable') else '0' }} net.netfilter.nf_conntrack_tcp_max_retrans = {{ tcp.max_retrans }} -net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }} \ No newline at end of file +net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }} +net.netfilter.nf_conntrack_timestamp = {{ '1' if log.timestamp is vyos_defined else '0' }} \ No newline at end of file diff --git a/debian/control b/debian/control index 189a959b0ed..c5497b7dd77 100644 --- a/debian/control +++ b/debian/control @@ -70,6 +70,7 @@ Depends: python3-netifaces, python3-paramiko, python3-passlib, + python3-pyroute2, python3-psutil, python3-pyhumps, python3-pystache, diff --git a/interface-definitions/include/conntrack/log-common.xml.i b/interface-definitions/include/conntrack/log-common.xml.i deleted file mode 100644 index 38799f8f4b3..00000000000 --- a/interface-definitions/include/conntrack/log-common.xml.i +++ /dev/null @@ -1,20 +0,0 @@ - - - - Log connection deletion - - - - - - Log connection creation - - - - - - Log connection updates - - - - diff --git a/interface-definitions/include/conntrack/log-protocols.xml.i b/interface-definitions/include/conntrack/log-protocols.xml.i new file mode 100644 index 00000000000..01925076039 --- /dev/null +++ b/interface-definitions/include/conntrack/log-protocols.xml.i @@ -0,0 +1,26 @@ + + + + Log connection tracking events for ICMP + + + + + + Log connection tracking events for all protocols other than TCP, UDP and ICMP + + + + + + Log connection tracking events for TCP + + + + + + Log connection tracking events for UDP + + + + diff --git a/interface-definitions/system_conntrack.xml.in b/interface-definitions/system_conntrack.xml.in index 0dfa2ea81b7..cd59d130815 100644 --- a/interface-definitions/system_conntrack.xml.in +++ b/interface-definitions/system_conntrack.xml.in @@ -223,41 +223,78 @@ - Log connection tracking events per protocol + Log connection tracking - + - Log connection tracking events for ICMP + Event type and protocol - #include + + + Log connection deletion + + + #include + + + + + Log connection creation + + + #include + + + + + Log connection updates + + + #include + + - + - Log connection tracking events for all protocols other than TCP, UDP and ICMP + Log connection tracking events include flow-based timestamp + - - #include - - - + + - Log connection tracking events for TCP + Internal message queue size + + u32:100-999999 + Queue size + + + + + Queue size must be between 100 and 999999 - - #include - - - + + - Log connection tracking events for UDP + Set log-level. Log must be enable. + + info debug + + + info + Info log level + + + debug + Debug log level + + + (info|debug) + - - #include - - + diff --git a/smoketest/scripts/cli/test_system_conntrack.py b/smoketest/scripts/cli/test_system_conntrack.py index 3ae7b6217a3..c07fdce778b 100755 --- a/smoketest/scripts/cli/test_system_conntrack.py +++ b/smoketest/scripts/cli/test_system_conntrack.py @@ -20,7 +20,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.firewall import find_nftables_rule -from vyos.utils.file import read_file +from vyos.utils.file import read_file, read_json base_path = ['system', 'conntrack'] @@ -28,6 +28,9 @@ def get_sysctl(parameter): tmp = parameter.replace(r'.', r'/') return read_file(f'/proc/sys/{tmp}') +def get_logger_config(): + return read_json('/run/vyos-conntrack-logger.conf') + class TestSystemConntrack(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -280,5 +283,35 @@ def test_conntrack_timeout_custom(self): self.verify_nftables(nftables6_search, 'ip6 vyos_conntrack') self.cli_delete(['firewall']) + + def test_conntrack_log(self): + expected_config = { + 'event': { + 'destroy': {}, + 'new': {}, + 'update': {}, + }, + 'queue_size': '10000' + } + self.cli_set(base_path + ['log', 'event', 'destroy']) + self.cli_set(base_path + ['log', 'event', 'new']) + self.cli_set(base_path + ['log', 'event', 'update']) + self.cli_set(base_path + ['log', 'queue-size', '10000']) + self.cli_commit() + self.assertEqual(expected_config, get_logger_config()) + self.assertEqual('0', get_sysctl('net.netfilter.nf_conntrack_timestamp')) + + for event in ['destroy', 'new', 'update']: + for proto in ['icmp', 'other', 'tcp', 'udp']: + self.cli_set(base_path + ['log', 'event', event, proto]) + expected_config['event'][event][proto] = {} + self.cli_set(base_path + ['log', 'timestamp']) + expected_config['timestamp'] = {} + self.cli_commit() + + self.assertEqual(expected_config, get_logger_config()) + self.assertEqual('1', get_sysctl('net.netfilter.nf_conntrack_timestamp')) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/system_conntrack.py b/src/conf_mode/system_conntrack.py index aa290788cb3..2529445bf92 100755 --- a/src/conf_mode/system_conntrack.py +++ b/src/conf_mode/system_conntrack.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +import json import os from sys import exit @@ -24,7 +24,8 @@ from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive -from vyos.utils.process import cmd +from vyos.utils.file import write_file +from vyos.utils.process import cmd, call from vyos.utils.process import rc_cmd from vyos.template import render from vyos import ConfigError @@ -34,6 +35,7 @@ conntrack_config = r'/etc/modprobe.d/vyatta_nf_conntrack.conf' sysctl_file = r'/run/sysctl/10-vyos-conntrack.conf' nftables_ct_file = r'/run/nftables-ct.conf' +vyos_conntrack_logger_config = r'/run/vyos-conntrack-logger.conf' # Every ALG (Application Layer Gateway) consists of either a Kernel Object # also called a Kernel Module/Driver or some rules present in iptables @@ -113,6 +115,7 @@ def get_config(config=None): return conntrack + def verify(conntrack): for inet in ['ipv4', 'ipv6']: if dict_search_args(conntrack, 'ignore', inet, 'rule') != None: @@ -181,6 +184,11 @@ def generate(conntrack): if not os.path.exists(nftables_ct_file): conntrack['first_install'] = True + if 'log' not in conntrack: + # Remove old conntrack-logger config and return + if os.path.exists(vyos_conntrack_logger_config): + os.unlink(vyos_conntrack_logger_config) + # Determine if conntrack is needed conntrack['ipv4_firewall_action'] = 'return' conntrack['ipv6_firewall_action'] = 'return' @@ -199,6 +207,11 @@ def generate(conntrack): render(conntrack_config, 'conntrack/vyos_nf_conntrack.conf.j2', conntrack) render(sysctl_file, 'conntrack/sysctl.conf.j2', conntrack) render(nftables_ct_file, 'conntrack/nftables-ct.j2', conntrack) + + if 'log' in conntrack: + log_conf_json = json.dumps(conntrack['log'], indent=4) + write_file(vyos_conntrack_logger_config, log_conf_json) + return None def apply(conntrack): @@ -243,8 +256,12 @@ def apply(conntrack): # See: https://bugzilla.redhat.com/show_bug.cgi?id=1264080 cmd(f'sysctl -f {sysctl_file}') + if 'log' in conntrack: + call(f'systemctl restart vyos-conntrack-logger.service') + return None + if __name__ == '__main__': try: c = get_config() diff --git a/src/services/vyos-conntrack-logger b/src/services/vyos-conntrack-logger new file mode 100755 index 00000000000..9c31b465f40 --- /dev/null +++ b/src/services/vyos-conntrack-logger @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import grp +import logging +import multiprocessing +import os +import queue +import signal +import socket +import threading +from datetime import timedelta +from pathlib import Path +from time import sleep +from typing import Dict, AnyStr + +from pyroute2 import conntrack +from pyroute2.netlink import nfnetlink +from pyroute2.netlink.nfnetlink import NFNL_SUBSYS_CTNETLINK +from pyroute2.netlink.nfnetlink.nfctsocket import nfct_msg, \ + IPCTNL_MSG_CT_DELETE, IPCTNL_MSG_CT_NEW, IPS_SEEN_REPLY, \ + IPS_OFFLOAD, IPS_ASSURED + +from vyos.utils.file import read_json + + +shutdown_event = multiprocessing.Event() + +logging.basicConfig(level=logging.INFO, format='%(message)s') +logger = logging.getLogger(__name__) + + +class DebugFormatter(logging.Formatter): + def format(self, record): + self._style._fmt = '[%(asctime)s] %(levelname)s: %(message)s' + return super().format(record) + + +def set_log_level(level: str) -> None: + if level == 'debug': + logger.setLevel(logging.DEBUG) + logger.parent.handlers[0].setFormatter(DebugFormatter()) + else: + logger.setLevel(logging.INFO) + + +EVENT_NAME_TO_GROUP = { + 'new': nfnetlink.NFNLGRP_CONNTRACK_NEW, + 'update': nfnetlink.NFNLGRP_CONNTRACK_UPDATE, + 'destroy': nfnetlink.NFNLGRP_CONNTRACK_DESTROY +} + +# https://github.com/torvalds/linux/blob/1dfe225e9af5bd3399a1dbc6a4df6a6041ff9c23/include/uapi/linux/netfilter/nf_conntrack_tcp.h#L9 +TCP_CONNTRACK_SYN_SENT = 1 +TCP_CONNTRACK_SYN_RECV = 2 +TCP_CONNTRACK_ESTABLISHED = 3 +TCP_CONNTRACK_FIN_WAIT = 4 +TCP_CONNTRACK_CLOSE_WAIT = 5 +TCP_CONNTRACK_LAST_ACK = 6 +TCP_CONNTRACK_TIME_WAIT = 7 +TCP_CONNTRACK_CLOSE = 8 +TCP_CONNTRACK_LISTEN = 9 +TCP_CONNTRACK_MAX = 10 +TCP_CONNTRACK_IGNORE = 11 +TCP_CONNTRACK_RETRANS = 12 +TCP_CONNTRACK_UNACK = 13 +TCP_CONNTRACK_TIMEOUT_MAX = 14 + +TCP_CONNTRACK_TO_NAME = { + TCP_CONNTRACK_SYN_SENT: "SYN_SENT", + TCP_CONNTRACK_SYN_RECV: "SYN_RECV", + TCP_CONNTRACK_ESTABLISHED: "ESTABLISHED", + TCP_CONNTRACK_FIN_WAIT: "FIN_WAIT", + TCP_CONNTRACK_CLOSE_WAIT: "CLOSE_WAIT", + TCP_CONNTRACK_LAST_ACK: "LAST_ACK", + TCP_CONNTRACK_TIME_WAIT: "TIME_WAIT", + TCP_CONNTRACK_CLOSE: "CLOSE", + TCP_CONNTRACK_LISTEN: "LISTEN", + TCP_CONNTRACK_MAX: "MAX", + TCP_CONNTRACK_IGNORE: "IGNORE", + TCP_CONNTRACK_RETRANS: "RETRANS", + TCP_CONNTRACK_UNACK: "UNACK", + TCP_CONNTRACK_TIMEOUT_MAX: "TIMEOUT_MAX", +} + +# https://github.com/torvalds/linux/blob/1dfe225e9af5bd3399a1dbc6a4df6a6041ff9c23/include/uapi/linux/netfilter/nf_conntrack_sctp.h#L8 +SCTP_CONNTRACK_CLOSED = 1 +SCTP_CONNTRACK_COOKIE_WAIT = 2 +SCTP_CONNTRACK_COOKIE_ECHOED = 3 +SCTP_CONNTRACK_ESTABLISHED = 4 +SCTP_CONNTRACK_SHUTDOWN_SENT = 5 +SCTP_CONNTRACK_SHUTDOWN_RECD = 6 +SCTP_CONNTRACK_SHUTDOWN_ACK_SENT = 7 +SCTP_CONNTRACK_HEARTBEAT_SENT = 8 +SCTP_CONNTRACK_HEARTBEAT_ACKED = 9 # no longer used +SCTP_CONNTRACK_MAX = 10 + +SCTP_CONNTRACK_TO_NAME = { + SCTP_CONNTRACK_CLOSED: 'CLOSED', + SCTP_CONNTRACK_COOKIE_WAIT: 'COOKIE_WAIT', + SCTP_CONNTRACK_COOKIE_ECHOED: 'COOKIE_ECHOED', + SCTP_CONNTRACK_ESTABLISHED: 'ESTABLISHED', + SCTP_CONNTRACK_SHUTDOWN_SENT: 'SHUTDOWN_SENT', + SCTP_CONNTRACK_SHUTDOWN_RECD: 'SHUTDOWN_RECD', + SCTP_CONNTRACK_SHUTDOWN_ACK_SENT: 'SHUTDOWN_ACK_SENT', + SCTP_CONNTRACK_HEARTBEAT_SENT: 'HEARTBEAT_SENT', + SCTP_CONNTRACK_HEARTBEAT_ACKED: 'HEARTBEAT_ACKED', + SCTP_CONNTRACK_MAX: 'MAX', +} + +PROTO_CONNTRACK_TO_NAME = { + 'TCP': TCP_CONNTRACK_TO_NAME, + 'SCTP': SCTP_CONNTRACK_TO_NAME +} + +SUPPORTED_PROTO_TO_NAME = { + socket.IPPROTO_ICMP: 'icmp', + socket.IPPROTO_TCP: 'tcp', + socket.IPPROTO_UDP: 'udp', +} + +PROTO_TO_NAME = { + socket.IPPROTO_ICMPV6: 'icmpv6', + socket.IPPROTO_SCTP: 'sctp', + socket.IPPROTO_GRE: 'gre', +} + +PROTO_TO_NAME.update(SUPPORTED_PROTO_TO_NAME) + + +def sig_handler(signum, frame): + process_name = multiprocessing.current_process().name + logger.debug(f'[{process_name}]: {"Shutdown" if signum == signal.SIGTERM else "Reload"} signal received...') + shutdown_event.set() + + +def format_flow_data(data: Dict) -> AnyStr: + """ + Formats the flow event data into a string suitable for logging. + """ + key_format = { + 'SRC_PORT': 'sport', + 'DST_PORT': 'dport' + } + message = f"src={data['ADDR'].get('SRC')} dst={data['ADDR'].get('DST')}" + + for key in ['SRC_PORT', 'DST_PORT', 'TYPE', 'CODE', 'ID']: + tmp = data['PROTO'].get(key) + if tmp is not None: + key = key_format.get(key, key) + message += f" {key.lower()}={tmp}" + + if 'COUNTERS' in data: + for key in ['PACKETS', 'BYTES']: + tmp = data['COUNTERS'].get(key) + if tmp is not None: + message += f" {key.lower()}={tmp}" + + return message + + +def format_event_message(event: Dict) -> AnyStr: + """ + Formats the internal parsed event data into a string suitable for logging. + """ + event_type = f"[{event['COMMON']['EVENT_TYPE'].upper()}]" + message = f"{event_type:<{9}} {event['COMMON']['ID']} " \ + f"{event['ORIG']['PROTO'].get('NAME'):<{8}} " \ + f"{event['ORIG']['PROTO'].get('NUMBER')} " + + tmp = event['COMMON']['TIME_OUT'] + if tmp is not None: message += f"{tmp} " + + if proto_info := event['COMMON'].get('PROTO_INFO'): + message += f"{proto_info.get('STATE_NAME')} " + + for key in ['ORIG', 'REPLY']: + message += f"{format_flow_data(event[key])} " + if key == 'ORIG' and not (event['COMMON']['STATUS'] & IPS_SEEN_REPLY): + message += f"[UNREPLIED] " + + tmp = event['COMMON']['MARK'] + if tmp is not None: message += f"mark={tmp} " + + if event['COMMON']['STATUS'] & IPS_OFFLOAD: message += f" [OFFLOAD] " + elif event['COMMON']['STATUS'] & IPS_ASSURED: message += f" [ASSURED] " + + if tmp := event['COMMON']['PORTID']: message += f"portid={tmp} " + if tstamp := event['COMMON'].get('TIMESTAMP'): + message += f"start={tstamp['START']} stop={tstamp['STOP']} " + delta_ns = tstamp['STOP'] - tstamp['START'] + delta_s = delta_ns // 1e9 + remaining_ns = delta_ns % 1e9 + delta = timedelta(seconds=delta_s, microseconds=remaining_ns / 1000) + message += f"delta={delta.total_seconds()} " + + return message + + +def parse_event_type(header: Dict) -> AnyStr: + """ + Extract event type from nfct_msg. new, update, destroy + """ + event_type = 'unknown' + if header['type'] == IPCTNL_MSG_CT_DELETE | (NFNL_SUBSYS_CTNETLINK << 8): + event_type = 'destroy' + elif header['type'] == IPCTNL_MSG_CT_NEW | (NFNL_SUBSYS_CTNETLINK << 8): + event_type = 'update' + if header['flags']: + event_type = 'new' + return event_type + + +def parse_proto(cta: nfct_msg.cta_tuple) -> Dict: + """ + Extract proto info from nfct_msg. src/dst port, code, type, id + """ + data = dict() + + cta_proto = cta.get_attr('CTA_TUPLE_PROTO') + proto_num = cta_proto.get_attr('CTA_PROTO_NUM') + + data['NUMBER'] = proto_num + data['NAME'] = PROTO_TO_NAME.get(proto_num, 'unknown') + + if proto_num in (socket.IPPROTO_ICMP, socket.IPPROTO_ICMPV6): + pref = 'CTA_PROTO_ICMP' + if proto_num == socket.IPPROTO_ICMPV6: pref += 'V6' + keys = ['TYPE', 'CODE', 'ID'] + else: + pref = 'CTA_PROTO' + keys = ['SRC_PORT', 'DST_PORT'] + + for key in keys: + data[key] = cta_proto.get_attr(f'{pref}_{key}') + + return data + + +def parse_proto_info(cta: nfct_msg.cta_protoinfo) -> Dict: + """ + Extract proto state and state name from nfct_msg + """ + data = dict() + if not cta: + return data + + for proto in ['TCP', 'SCTP']: + if proto_info := cta.get_attr(f'CTA_PROTOINFO_{proto}'): + data['STATE'] = proto_info.get_attr(f'CTA_PROTOINFO_{proto}_STATE') + data['STATE_NAME'] = PROTO_CONNTRACK_TO_NAME.get(proto, {}).get(data['STATE'], 'unknown') + return data + + +def parse_timestamp(cta: nfct_msg.cta_timestamp) -> Dict: + """ + Extract timestamp from nfct_msg + """ + data = dict() + if not cta: + return data + data['START'] = cta.get_attr('CTA_TIMESTAMP_START') + data['STOP'] = cta.get_attr('CTA_TIMESTAMP_STOP') + + return data + + +def parse_ip_addr(family: int, cta: nfct_msg.cta_tuple) -> Dict: + """ + Extract ip adr from nfct_msg + """ + data = dict() + cta_ip = cta.get_attr('CTA_TUPLE_IP') + + if family == socket.AF_INET: + pref = 'CTA_IP_V4' + elif family == socket.AF_INET6: + pref = 'CTA_IP_V6' + else: + logger.error(f'Undefined INET: {family}') + raise NotImplementedError(family) + + for direct in ['SRC', 'DST']: + data[direct] = cta_ip.get_attr(f'{pref}_{direct}') + + return data + + +def parse_counters(cta: nfct_msg.cta_counters) -> Dict: + """ + Extract counters from nfct_msg + """ + data = dict() + if not cta: + return data + + for key in ['PACKETS', 'BYTES']: + tmp = cta.get_attr(f'CTA_COUNTERS_{key}') + if tmp is None: + tmp = cta.get_attr(f'CTA_COUNTERS32_{key}') + data['key'] = tmp + + return data + + +def is_need_to_log(event_type: AnyStr, proto_num: int, conf_event: Dict): + """ + Filter message by event type and protocols + """ + conf = conf_event.get(event_type) + if conf == {} or conf.get(SUPPORTED_PROTO_TO_NAME.get(proto_num, 'other')) is not None: + return True + return False + + +def parse_conntrack_event(msg: nfct_msg, conf_event: Dict) -> Dict: + """ + Convert nfct_msg to internal data dict. + """ + data = dict() + event_type = parse_event_type(msg['header']) + proto_num = msg.get_nested('CTA_TUPLE_ORIG', 'CTA_TUPLE_PROTO', 'CTA_PROTO_NUM') + + if not is_need_to_log(event_type, proto_num, conf_event): + return data + + data = { + 'COMMON': { + 'ID': msg.get_attr('CTA_ID'), + 'EVENT_TYPE': event_type, + 'TIME_OUT': msg.get_attr('CTA_TIMEOUT'), + 'MARK': msg.get_attr('CTA_MARK'), + 'PORTID': msg['header'].get('pid'), + 'PROTO_INFO': parse_proto_info(msg.get_attr('CTA_PROTOINFO')), + 'STATUS': msg.get_attr('CTA_STATUS'), + 'TIMESTAMP': parse_timestamp(msg.get_attr('CTA_TIMESTAMP')) + }, + 'ORIG': {}, + 'REPLY': {}, + } + + for direct in ['ORIG', 'REPLY']: + data[direct]['ADDR'] = parse_ip_addr(msg['nfgen_family'], msg.get_attr(f'CTA_TUPLE_{direct}')) + data[direct]['PROTO'] = parse_proto(msg.get_attr(f'CTA_TUPLE_{direct}')) + data[direct]['COUNTERS'] = parse_counters(msg.get_attr(f'CTA_COUNTERS_{direct}')) + + return data + + +def worker(ct: conntrack.Conntrack, shutdown_event: multiprocessing.Event, conf_event: Dict): + """ + Main function of parser worker process + """ + process_name = multiprocessing.current_process().name + logger.debug(f'[{process_name}] started') + timeout = 0.1 + while not shutdown_event.is_set(): + if not ct.buffer_queue.empty(): + try: + for msg in ct.get(): + parsed_event = parse_conntrack_event(msg, conf_event) + if parsed_event: + message = format_event_message(parsed_event) + if logger.level == logging.DEBUG: + logger.debug(f"[{process_name}]: {message} raw: {msg}") + else: + logger.info(message) + except queue.Full: + logger.error("Conntrack message queue if full.") + except Exception as e: + logger.error(f"Error in queue: {e.__class__} {e}") + else: + sleep(timeout) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--config', + action='store', + help='Path to vyos-conntrack-logger configuration', + required=True, + type=Path) + + args = parser.parse_args() + try: + config = read_json(args.config) + except Exception as err: + logger.error(f'Configuration file "{args.config}" does not exist or malformed: {err}') + exit(1) + + set_log_level(config.get('log_level', 'info')) + + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGTERM, sig_handler) + + if 'event' in config: + event_groups = list(config.get('event').keys()) + else: + logger.error(f'Configuration is wrong. Event filter is empty.') + exit(1) + + conf_event = config['event'] + qsize = config.get('queue_size') + ct = conntrack.Conntrack(async_qsize=int(qsize) if qsize else None) + ct.buffer_queue = multiprocessing.Queue(ct.async_qsize) + ct.bind(async_cache=True) + + for name in event_groups: + if group := EVENT_NAME_TO_GROUP.get(name): + ct.add_membership(group) + else: + logger.error(f'Unexpected event group {name}') + processes = list() + try: + for _ in range(multiprocessing.cpu_count()): + p = multiprocessing.Process(target=worker, args=(ct, + shutdown_event, + conf_event)) + processes.append(p) + p.start() + logger.info('Conntrack socket bound and listening for messages.') + + while not shutdown_event.is_set(): + if not ct.pthread.is_alive(): + if ct.buffer_queue.qsize()/ct.async_qsize < 0.9: + if not shutdown_event.is_set(): + logger.debug('Restart listener thread') + # restart listener thread after queue overloaded when queue size low than 90% + ct.pthread = threading.Thread( + name="Netlink async cache", target=ct.async_recv + ) + ct.pthread.daemon = True + ct.pthread.start() + else: + sleep(0.1) + finally: + for p in processes: + p.join() + if not p.is_alive(): + logger.debug(f"[{p.name}]: finished") + ct.close() + logging.info("Conntrack socket closed.") + exit() diff --git a/src/systemd/vyos-conntrack-logger.service b/src/systemd/vyos-conntrack-logger.service new file mode 100644 index 00000000000..9bc1d857b7f --- /dev/null +++ b/src/systemd/vyos-conntrack-logger.service @@ -0,0 +1,21 @@ +[Unit] +Description=VyOS conntrack logger daemon + +# Seemingly sensible way to say "as early as the system is ready" +# All vyos-configd needs is read/write mounted root +After=conntrackd.service + +[Service] +ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/services/vyos-conntrack-logger -c /run/vyos-conntrack-logger.conf +Type=idle + +SyslogIdentifier=vyos-conntrack-logger +SyslogFacility=daemon + +Restart=on-failure + +User=root +Group=vyattacfg + +[Install] +WantedBy=multi-user.target From 4d2c89dcd50d3c158dc76ac5ab843dd66105bc02 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Thu, 28 Dec 2023 22:26:56 -0800 Subject: [PATCH 03/64] T5873: vpn ipsec remote-access: support VTI interfaces --- data/templates/ipsec/swanctl.conf.j2 | 2 + data/templates/ipsec/swanctl/remote_access.j2 | 7 + .../include/ipsec/bind.xml.i | 10 + interface-definitions/vpn_ipsec.xml.in | 49 +++- smoketest/scripts/cli/test_vpn_ipsec.py | 256 ++++++++++++++++++ src/conf_mode/vpn_ipsec.py | 74 ++++- 6 files changed, 382 insertions(+), 16 deletions(-) create mode 100644 interface-definitions/include/ipsec/bind.xml.i diff --git a/data/templates/ipsec/swanctl.conf.j2 b/data/templates/ipsec/swanctl.conf.j2 index d44d0f5e47b..698a9135efa 100644 --- a/data/templates/ipsec/swanctl.conf.j2 +++ b/data/templates/ipsec/swanctl.conf.j2 @@ -31,6 +31,8 @@ pools { {{ pool }} { {% if pool_config.prefix is vyos_defined %} addrs = {{ pool_config.prefix }} +{% elif pool_config.range is vyos_defined %} + addrs = {{ pool_config.range.start }}-{{ pool_config.range.stop }} {% endif %} {% if pool_config.name_server is vyos_defined %} dns = {{ pool_config.name_server | join(',') }} diff --git a/data/templates/ipsec/swanctl/remote_access.j2 b/data/templates/ipsec/swanctl/remote_access.j2 index e384ae9720b..a3b61f781ac 100644 --- a/data/templates/ipsec/swanctl/remote_access.j2 +++ b/data/templates/ipsec/swanctl/remote_access.j2 @@ -69,6 +69,13 @@ {% set local_port = rw_conf.local.port if rw_conf.local.port is vyos_defined else '' %} {% set local_suffix = '[%any/{1}]'.format(local_port) if local_port else '' %} local_ts = {{ local_prefix | join(local_suffix + ",") }}{{ local_suffix }} +{% if rw_conf.bind is vyos_defined %} +{# The key defaults to 0 and will match any policies which similarly do not have a lookup key configuration. #} +{# Thus we simply shift the key by one to also support a vti0 interface #} +{% set if_id = rw_conf.bind | replace('vti', '') | int + 1 %} + if_id_in = {{ if_id }} + if_id_out = {{ if_id }} +{% endif %} } } } diff --git a/interface-definitions/include/ipsec/bind.xml.i b/interface-definitions/include/ipsec/bind.xml.i new file mode 100644 index 00000000000..edc46d403de --- /dev/null +++ b/interface-definitions/include/ipsec/bind.xml.i @@ -0,0 +1,10 @@ + + + + VTI tunnel interface associated with this configuration + + interfaces vti + + + + diff --git a/interface-definitions/vpn_ipsec.xml.in b/interface-definitions/vpn_ipsec.xml.in index 4a7fde75bbd..d9d6fd93bb7 100644 --- a/interface-definitions/vpn_ipsec.xml.in +++ b/interface-definitions/vpn_ipsec.xml.in @@ -854,6 +854,7 @@ #include #include #include + #include Timeout to close connection if no data is transmitted @@ -978,6 +979,45 @@ + + + Local IPv4 or IPv6 pool range + + + + + First IP address for local pool range + + ipv4 + IPv4 start address of pool + + + ipv6 + IPv6 start address of pool + + + + + + + + + Last IP address for local pool range + + ipv4 + IPv4 end address of pool + + + ipv6 + IPv6 end address of pool + + + + + + + + #include @@ -1201,14 +1241,7 @@ Virtual tunnel interface - - - VTI tunnel interface associated with this configuration - - interfaces vti - - - + #include #include diff --git a/smoketest/scripts/cli/test_vpn_ipsec.py b/smoketest/scripts/cli/test_vpn_ipsec.py index 2dc66485b58..2674b37b66a 100755 --- a/smoketest/scripts/cli/test_vpn_ipsec.py +++ b/smoketest/scripts/cli/test_vpn_ipsec.py @@ -1086,5 +1086,261 @@ def test_remote_access_no_rekey(self): self.tearDownPKI() + def test_remote_access_pool_range(self): + # Same as test_remote_access but using an IP pool range instead of prefix + self.setupPKI() + + ike_group = 'IKE-RW' + esp_group = 'ESP-RW' + + conn_name = 'vyos-rw' + local_address = '192.0.2.1' + ip_pool_name = 'ra-rw-ipv4' + username = 'vyos' + password = 'secret' + ike_lifetime = '7200' + eap_lifetime = '3600' + local_id = 'ipsec.vyos.net' + + name_servers = ['172.16.254.100', '172.16.254.101'] + range_start = '172.16.250.2' + range_stop = '172.16.250.254' + + # IKE + self.cli_set(base_path + ['ike-group', ike_group, 'key-exchange', 'ikev2']) + self.cli_set(base_path + ['ike-group', ike_group, 'lifetime', ike_lifetime]) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'hash', 'sha512']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'hash', 'sha256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'dh-group', '2']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'hash', 'sha256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'encryption', 'aes128gcm128']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'hash', 'sha256']) + + # ESP + self.cli_set(base_path + ['esp-group', esp_group, 'lifetime', eap_lifetime]) + self.cli_set(base_path + ['esp-group', esp_group, 'pfs', 'disable']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '1', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '1', 'hash', 'sha512']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '2', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '2', 'hash', 'sha384']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '3', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '3', 'hash', 'sha256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '4', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '4', 'hash', 'sha1']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '10', 'encryption', 'aes128gcm128']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '10', 'hash', 'sha256']) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'local-id', local_id]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'local-users', 'username', username, 'password', password]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'server-mode', 'x509']) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'certificate', peer_name]) + # verify() - CA cert required for x509 auth + with self.assertRaises(ConfigSessionError): + self.cli_commit() + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'ca-certificate', ca_name]) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'esp-group', esp_group]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'ike-group', ike_group]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'local-address', local_address]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'pool', ip_pool_name]) + + for ns in name_servers: + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'name-server', ns]) + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'range', 'start', range_start]) + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'range', 'stop', range_stop]) + + self.cli_commit() + + # verify applied configuration + swanctl_conf = read_file(swanctl_file) + swanctl_lines = [ + f'{conn_name}', + f'remote_addrs = %any', + f'local_addrs = {local_address}', + f'proposals = aes256-sha512-modp2048,aes256-sha256-modp2048,aes256-sha256-modp1024,aes128gcm128-sha256-modp2048', + f'version = 2', + f'send_certreq = no', + f'rekey_time = {ike_lifetime}s', + f'keyingtries = 0', + f'pools = {ip_pool_name}', + f'id = "{local_id}"', + f'auth = pubkey', + f'certs = peer1.pem', + f'auth = eap-mschapv2', + f'eap_id = %any', + f'esp_proposals = aes256-sha512,aes256-sha384,aes256-sha256,aes256-sha1,aes128gcm128-sha256', + f'life_time = {eap_lifetime}s', + f'dpd_action = clear', + f'replay_window = 32', + f'inactivity = 28800', + f'local_ts = 0.0.0.0/0,::/0', + ] + for line in swanctl_lines: + self.assertIn(line, swanctl_conf) + + swanctl_secrets_lines = [ + f'eap-{conn_name}-{username}', + f'secret = "{password}"', + f'id-{conn_name}-{username} = "{username}"', + ] + for line in swanctl_secrets_lines: + self.assertIn(line, swanctl_conf) + + swanctl_pool_lines = [ + f'{ip_pool_name}', + f'addrs = {range_start}-{range_stop}', + f'dns = {",".join(name_servers)}', + ] + for line in swanctl_pool_lines: + self.assertIn(line, swanctl_conf) + + # Check Root CA, Intermediate CA and Peer cert/key pair is present + self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{ca_name}.pem'))) + self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) + + self.tearDownPKI() + + def test_remote_access_vti(self): + # Set up and use a VTI interface for the remote access VPN + self.setupPKI() + + ike_group = 'IKE-RW' + esp_group = 'ESP-RW' + + conn_name = 'vyos-rw' + local_address = '192.0.2.1' + vti = 'vti10' + ip_pool_name = 'ra-rw-ipv4' + username = 'vyos' + password = 'secret' + ike_lifetime = '7200' + eap_lifetime = '3600' + local_id = 'ipsec.vyos.net' + + name_servers = ['10.1.1.1'] + range_start = '10.1.1.10' + range_stop = '10.1.1.254' + + # VTI interface + self.cli_set(vti_path + [vti, 'address', '10.1.1.1/24']) + + # IKE + self.cli_set(base_path + ['ike-group', ike_group, 'key-exchange', 'ikev2']) + self.cli_set(base_path + ['ike-group', ike_group, 'lifetime', ike_lifetime]) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '1', 'hash', 'sha512']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '2', 'hash', 'sha256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'dh-group', '2']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'encryption', 'aes256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '3', 'hash', 'sha256']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'dh-group', '14']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'encryption', 'aes128gcm128']) + self.cli_set(base_path + ['ike-group', ike_group, 'proposal', '10', 'hash', 'sha256']) + + # ESP + self.cli_set(base_path + ['esp-group', esp_group, 'lifetime', eap_lifetime]) + self.cli_set(base_path + ['esp-group', esp_group, 'pfs', 'disable']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '1', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '1', 'hash', 'sha512']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '2', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '2', 'hash', 'sha384']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '3', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '3', 'hash', 'sha256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '4', 'encryption', 'aes256']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '4', 'hash', 'sha1']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '10', 'encryption', 'aes128gcm128']) + self.cli_set(base_path + ['esp-group', esp_group, 'proposal', '10', 'hash', 'sha256']) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'local-id', local_id]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'local-users', 'username', username, 'password', password]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'server-mode', 'x509']) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'certificate', peer_name]) + # verify() - CA cert required for x509 auth + with self.assertRaises(ConfigSessionError): + self.cli_commit() + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'ca-certificate', ca_name]) + + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'esp-group', esp_group]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'ike-group', ike_group]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'bind', vti]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'local-address', local_address]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'pool', ip_pool_name]) + + for ns in name_servers: + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'name-server', ns]) + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'range', 'start', range_start]) + self.cli_set(base_path + ['remote-access', 'pool', ip_pool_name, 'range', 'stop', range_stop]) + + self.cli_commit() + + # verify applied configuration + swanctl_conf = read_file(swanctl_file) + + if_id = vti.lstrip('vti') + # The key defaults to 0 and will match any policies which similarly do + # not have a lookup key configuration - thus we shift the key by one + # to also support a vti0 interface + if_id = str(int(if_id) +1) + + swanctl_lines = [ + f'{conn_name}', + f'remote_addrs = %any', + f'local_addrs = {local_address}', + f'proposals = aes256-sha512-modp2048,aes256-sha256-modp2048,aes256-sha256-modp1024,aes128gcm128-sha256-modp2048', + f'version = 2', + f'send_certreq = no', + f'rekey_time = {ike_lifetime}s', + f'keyingtries = 0', + f'pools = {ip_pool_name}', + f'id = "{local_id}"', + f'auth = pubkey', + f'certs = peer1.pem', + f'auth = eap-mschapv2', + f'eap_id = %any', + f'esp_proposals = aes256-sha512,aes256-sha384,aes256-sha256,aes256-sha1,aes128gcm128-sha256', + f'life_time = {eap_lifetime}s', + f'dpd_action = clear', + f'replay_window = 32', + f'if_id_in = {if_id}', # will be 11 for vti10 - shifted by one + f'if_id_out = {if_id}', + f'inactivity = 28800', + f'local_ts = 0.0.0.0/0,::/0', + ] + for line in swanctl_lines: + self.assertIn(line, swanctl_conf) + + swanctl_secrets_lines = [ + f'eap-{conn_name}-{username}', + f'secret = "{password}"', + f'id-{conn_name}-{username} = "{username}"', + ] + for line in swanctl_secrets_lines: + self.assertIn(line, swanctl_conf) + + swanctl_pool_lines = [ + f'{ip_pool_name}', + f'addrs = {range_start}-{range_stop}', + f'dns = {",".join(name_servers)}', + ] + for line in swanctl_pool_lines: + self.assertIn(line, swanctl_conf) + + # Check Root CA, Intermediate CA and Peer cert/key pair is present + self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{ca_name}.pem'))) + self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) + + self.tearDownPKI() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index cf82b767f85..2c1ddc245f1 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -21,6 +21,9 @@ from sys import exit from time import sleep +from ipaddress import ip_address +from netaddr import IPNetwork +from netaddr import IPRange from vyos.base import Warning from vyos.config import Config @@ -304,6 +307,14 @@ def verify(ipsec): if dict_search('remote_access.radius.server', ipsec) == None: raise ConfigError('RADIUS authentication requires at least one server') + if 'bind' in ra_conf: + if dict_search('options.disable_route_autoinstall', ipsec) == None: + Warning('It\'s recommended to use ipsec vti with the next command\n[set vpn ipsec option disable-route-autoinstall]') + + vti_interface = ra_conf['bind'] + if not os.path.exists(f'/sys/class/net/{vti_interface}'): + raise ConfigError(f'VTI interface {vti_interface} for remote-access connection {name} does not exist!') + if 'pool' in ra_conf: if {'dhcp', 'radius'} <= set(ra_conf['pool']): raise ConfigError(f'Can not use both DHCP and RADIUS for address allocation '\ @@ -330,26 +341,73 @@ def verify(ipsec): raise ConfigError(f'Requested pool "{pool}" does not exist!') if 'pool' in ipsec['remote_access']: + pool_networks = [] for pool, pool_config in ipsec['remote_access']['pool'].items(): - if 'prefix' not in pool_config: - raise ConfigError(f'Missing madatory prefix option for pool "{pool}"!') + if 'prefix' not in pool_config and 'range' not in pool_config: + raise ConfigError(f'Mandatory prefix or range must be specified for pool "{pool}"!') + + if 'prefix' in pool_config and 'range' in pool_config: + raise ConfigError(f'Only one of prefix or range can be specified for pool "{pool}"!') + + if 'prefix' in pool_config: + range_is_ipv4 = is_ipv4(pool_config['prefix']) + range_is_ipv6 = is_ipv6(pool_config['prefix']) + + net = IPNetwork(pool_config['prefix']) + start = net.first + stop = net.last + for network in pool_networks: + if start in network or stop in network: + raise ConfigError(f'Prefix for pool "{pool}" is already part of another pool\'s range!') + + tmp = IPRange(start, stop) + pool_networks.append(tmp) + + if 'range' in pool_config: + range_config = pool_config['range'] + if not {'start', 'stop'} <= set(range_config.keys()): + raise ConfigError(f'Range start and stop address must be defined for pool "{pool}"!') + + range_both_ipv4 = is_ipv4(range_config['start']) and is_ipv4(range_config['stop']) + range_both_ipv6 = is_ipv6(range_config['start']) and is_ipv6(range_config['stop']) + + if not (range_both_ipv4 or range_both_ipv6): + raise ConfigError(f'Range start and stop must be of the same address family for pool "{pool}"!') + + if ip_address(range_config['stop']) < ip_address(range_config['start']): + raise ConfigError(f'Range stop address must be greater or equal\n' \ + 'to the range\'s start address for pool "{pool}"!') + + range_is_ipv4 = is_ipv4(range_config['start']) + range_is_ipv6 = is_ipv6(range_config['start']) + + start = range_config['start'] + stop = range_config['stop'] + for network in pool_networks: + if start in network: + raise ConfigError(f'Range "{range}" start address "{start}" already part of another pool\'s range!') + if stop in network: + raise ConfigError(f'Range "{range}" stop address "{stop}" already part of another pool\'s range!') + + tmp = IPRange(start, stop) + pool_networks.append(tmp) if 'name_server' in pool_config: if len(pool_config['name_server']) > 2: raise ConfigError(f'Only two name-servers are supported for remote-access pool "{pool}"!') for ns in pool_config['name_server']: - v4_addr_and_ns = is_ipv4(ns) and not is_ipv4(pool_config['prefix']) - v6_addr_and_ns = is_ipv6(ns) and not is_ipv6(pool_config['prefix']) + v4_addr_and_ns = is_ipv4(ns) and not range_is_ipv4 + v6_addr_and_ns = is_ipv6(ns) and not range_is_ipv6 if v4_addr_and_ns or v6_addr_and_ns: - raise ConfigError('Must use both IPv4 or IPv6 addresses for pool prefix and name-server adresses!') + raise ConfigError('Must use both IPv4 or IPv6 addresses for pool prefix/range and name-server addresses!') if 'exclude' in pool_config: for exclude in pool_config['exclude']: - v4_addr_and_exclude = is_ipv4(exclude) and not is_ipv4(pool_config['prefix']) - v6_addr_and_exclude = is_ipv6(exclude) and not is_ipv6(pool_config['prefix']) + v4_addr_and_exclude = is_ipv4(exclude) and not range_is_ipv4 + v6_addr_and_exclude = is_ipv6(exclude) and not range_is_ipv6 if v4_addr_and_exclude or v6_addr_and_exclude: - raise ConfigError('Must use both IPv4 or IPv6 addresses for pool prefix and exclude prefixes!') + raise ConfigError('Must use both IPv4 or IPv6 addresses for pool prefix/range and exclude prefixes!') if 'radius' in ipsec['remote_access'] and 'server' in ipsec['remote_access']['radius']: for server, server_config in ipsec['remote_access']['radius']['server'].items(): From b62b2f5f8a9c4f0a7dc26bce1f15843651119256 Mon Sep 17 00:00:00 2001 From: srividya0208 Date: Mon, 15 Jul 2024 06:30:00 -0400 Subject: [PATCH 04/64] OpenVPN CLI-option: T6571: rename ncp-ciphers with data-ciphers --- data/templates/openvpn/server.conf.j2 | 4 +-- .../include/version/openvpn-version.xml.i | 2 +- .../interfaces_openvpn.xml.in | 2 +- python/vyos/template.py | 4 +-- .../config-tests/dialup-router-medium-vpn | 6 ++-- .../scripts/cli/test_interfaces_openvpn.py | 10 +++---- src/conf_mode/interfaces_openvpn.py | 6 ++-- src/migration-scripts/openvpn/3-to-4 | 30 +++++++++++++++++++ 8 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 src/migration-scripts/openvpn/3-to-4 diff --git a/data/templates/openvpn/server.conf.j2 b/data/templates/openvpn/server.conf.j2 index 6ac52544379..f6951969746 100644 --- a/data/templates/openvpn/server.conf.j2 +++ b/data/templates/openvpn/server.conf.j2 @@ -206,8 +206,8 @@ tls-server {% if encryption.cipher is vyos_defined %} cipher {{ encryption.cipher | openvpn_cipher }} {% endif %} -{% if encryption.ncp_ciphers is vyos_defined %} -data-ciphers {{ encryption.ncp_ciphers | openvpn_ncp_ciphers }} +{% if encryption.data_ciphers is vyos_defined %} +data-ciphers {{ encryption.data_ciphers | openvpn_data_ciphers }} {% endif %} {% endif %} providers default diff --git a/interface-definitions/include/version/openvpn-version.xml.i b/interface-definitions/include/version/openvpn-version.xml.i index e03ad55c082..67ef2198347 100644 --- a/interface-definitions/include/version/openvpn-version.xml.i +++ b/interface-definitions/include/version/openvpn-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/interfaces_openvpn.xml.in b/interface-definitions/interfaces_openvpn.xml.in index 1860523c2bf..13ef3ae5bc4 100644 --- a/interface-definitions/interfaces_openvpn.xml.in +++ b/interface-definitions/interfaces_openvpn.xml.in @@ -87,7 +87,7 @@ - + Cipher negotiation list for use in server or client mode diff --git a/python/vyos/template.py b/python/vyos/template.py index e8d7ba669f2..3507e0940f7 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -556,8 +556,8 @@ def get_openvpn_cipher(cipher): return openvpn_translate[cipher].upper() return cipher.upper() -@register_filter('openvpn_ncp_ciphers') -def get_openvpn_ncp_ciphers(ciphers): +@register_filter('openvpn_data_ciphers') +def get_openvpn_data_ciphers(ciphers): out = [] for cipher in ciphers: if cipher in openvpn_translate: diff --git a/smoketest/config-tests/dialup-router-medium-vpn b/smoketest/config-tests/dialup-router-medium-vpn index 67af456f469..d6b00c6783d 100644 --- a/smoketest/config-tests/dialup-router-medium-vpn +++ b/smoketest/config-tests/dialup-router-medium-vpn @@ -33,7 +33,7 @@ set interfaces ethernet eth1 mtu '9000' set interfaces ethernet eth1 offload gro set interfaces ethernet eth1 speed 'auto' set interfaces loopback lo -set interfaces openvpn vtun0 encryption ncp-ciphers 'aes256' +set interfaces openvpn vtun0 encryption data-ciphers 'aes256' set interfaces openvpn vtun0 hash 'sha512' set interfaces openvpn vtun0 ip adjust-mss '1380' set interfaces openvpn vtun0 ip source-validation 'strict' @@ -52,7 +52,7 @@ set interfaces openvpn vtun0 tls ca-certificate 'openvpn_vtun0_2' set interfaces openvpn vtun0 tls certificate 'openvpn_vtun0' set interfaces openvpn vtun1 authentication password 'vyos1' set interfaces openvpn vtun1 authentication username 'vyos1' -set interfaces openvpn vtun1 encryption ncp-ciphers 'aes256' +set interfaces openvpn vtun1 encryption data-ciphers 'aes256' set interfaces openvpn vtun1 hash 'sha1' set interfaces openvpn vtun1 ip adjust-mss '1380' set interfaces openvpn vtun1 keep-alive failure-count '3' @@ -77,7 +77,7 @@ set interfaces openvpn vtun1 tls ca-certificate 'openvpn_vtun1_2' set interfaces openvpn vtun2 authentication password 'vyos2' set interfaces openvpn vtun2 authentication username 'vyos2' set interfaces openvpn vtun2 disable -set interfaces openvpn vtun2 encryption ncp-ciphers 'aes256' +set interfaces openvpn vtun2 encryption data-ciphers 'aes256' set interfaces openvpn vtun2 hash 'sha512' set interfaces openvpn vtun2 ip adjust-mss '1380' set interfaces openvpn vtun2 keep-alive failure-count '3' diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index 9ca661e8726..ca47c32181d 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -123,7 +123,7 @@ def test_openvpn_client_verify(self): interface = 'vtun2000' path = base_path + [interface] self.cli_set(path + ['mode', 'client']) - self.cli_set(path + ['encryption', 'ncp-ciphers', 'aes192gcm']) + self.cli_set(path + ['encryption', 'data-ciphers', 'aes192gcm']) # check validate() - cannot specify local-port in client mode self.cli_set(path + ['local-port', '5000']) @@ -197,7 +197,7 @@ def test_openvpn_client_interfaces(self): auth_hash = 'sha1' self.cli_set(path + ['device-type', 'tun']) - self.cli_set(path + ['encryption', 'ncp-ciphers', 'aes256']) + self.cli_set(path + ['encryption', 'data-ciphers', 'aes256']) self.cli_set(path + ['hash', auth_hash]) self.cli_set(path + ['mode', 'client']) self.cli_set(path + ['persistent-tunnel']) @@ -371,7 +371,7 @@ def test_openvpn_server_subnet_topology(self): port = str(2000 + ii) self.cli_set(path + ['device-type', 'tun']) - self.cli_set(path + ['encryption', 'ncp-ciphers', 'aes192']) + self.cli_set(path + ['encryption', 'data-ciphers', 'aes192']) self.cli_set(path + ['hash', auth_hash]) self.cli_set(path + ['mode', 'server']) self.cli_set(path + ['local-port', port]) @@ -462,8 +462,8 @@ def test_openvpn_site2site_verify(self): self.cli_set(path + ['mode', 'site-to-site']) - # check validate() - encryption ncp-ciphers cannot be specified in site-to-site mode - self.cli_set(path + ['encryption', 'ncp-ciphers', 'aes192gcm']) + # check validate() - cipher negotiation cannot be enabled in site-to-site mode + self.cli_set(path + ['encryption', 'data-ciphers', 'aes192gcm']) with self.assertRaises(ConfigSessionError): self.cli_commit() self.cli_delete(path + ['encryption']) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 320ab7b7b52..a03bd595989 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -322,8 +322,8 @@ def verify(openvpn): if v4addr in openvpn['local_address'] and 'subnet_mask' not in openvpn['local_address'][v4addr]: raise ConfigError('Must specify IPv4 "subnet-mask" for local-address') - if dict_search('encryption.ncp_ciphers', openvpn): - raise ConfigError('NCP ciphers can only be used in client or server mode') + if dict_search('encryption.data_ciphers', openvpn): + raise ConfigError('Cipher negotiation can only be used in client or server mode') else: # checks for client-server or site-to-site bridged @@ -520,7 +520,7 @@ def verify(openvpn): if dict_search('encryption.cipher', openvpn): raise ConfigError('"encryption cipher" option is deprecated for TLS mode. ' - 'Use "encryption ncp-ciphers" instead') + 'Use "encryption data-ciphers" instead') if dict_search('encryption.cipher', openvpn) == 'none': print('Warning: "encryption none" was specified!') diff --git a/src/migration-scripts/openvpn/3-to-4 b/src/migration-scripts/openvpn/3-to-4 new file mode 100644 index 00000000000..d3c76c7d332 --- /dev/null +++ b/src/migration-scripts/openvpn/3-to-4 @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . +# Renames ncp-ciphers option to data-ciphers + +from vyos.configtree import ConfigTree + +def migrate(config: ConfigTree) -> None: + if not config.exists(['interfaces', 'openvpn']): + # Nothing to do + return + + ovpn_intfs = config.list_nodes(['interfaces', 'openvpn']) + for i in ovpn_intfs: + #Rename 'encryption ncp-ciphers' with 'encryption data-ciphers' + ncp_cipher_path = ['interfaces', 'openvpn', i, 'encryption', 'ncp-ciphers'] + if config.exists(ncp_cipher_path): + config.rename(ncp_cipher_path, 'data-ciphers') From d6e9824f1612bd8c876437c071f31a1a0f44af5d Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Fri, 26 Jul 2024 13:25:19 +0200 Subject: [PATCH 05/64] vrf: T6603: conntrack ct_iface_map must only contain one entry for iifname/oifname When any of the following features NAT, NAT66 or Firewall is enabled, for every VRF on the CLI we install one rule into nftables for conntrack: chain vrf_zones_ct_in { type filter hook prerouting priority raw; policy accept; counter packets 3113 bytes 32227 ct original zone set iifname map @ct_iface_map counter packets 8550 bytes 80739 ct original zone set iifname map @ct_iface_map counter packets 5644 bytes 67697 ct original zone set iifname map @ct_iface_map } This is superfluous. --- smoketest/scripts/cli/test_vrf.py | 22 +++++++++++++++++++--- src/conf_mode/vrf.py | 12 +++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/smoketest/scripts/cli/test_vrf.py b/smoketest/scripts/cli/test_vrf.py index 176882ca510..2bb6c91c144 100755 --- a/smoketest/scripts/cli/test_vrf.py +++ b/smoketest/scripts/cli/test_vrf.py @@ -19,6 +19,8 @@ import unittest from base_vyostest_shim import VyOSUnitTestSHIM +from json import loads +from jmespath import search from vyos.configsession import ConfigSessionError from vyos.ifconfig import Interface @@ -28,6 +30,7 @@ from vyos.utils.network import get_vrf_tableid from vyos.utils.network import is_intf_addr_assigned from vyos.utils.network import interface_exists +from vyos.utils.process import cmd from vyos.utils.system import sysctl_read base_path = ['vrf'] @@ -557,26 +560,39 @@ def test_vrf_ip_ipv6_nht(self): self.assertNotIn(f' no ipv6 nht resolve-via-default', frrconfig) def test_vrf_conntrack(self): - table = '1000' + table = '8710' nftables_rules = { 'vrf_zones_ct_in': ['ct original zone set iifname map @ct_iface_map'], 'vrf_zones_ct_out': ['ct original zone set oifname map @ct_iface_map'] } - self.cli_set(base_path + ['name', 'blue', 'table', table]) + self.cli_set(base_path + ['name', 'randomVRF', 'table', '1000']) self.cli_commit() # Conntrack rules should not be present for chain, rule in nftables_rules.items(): self.verify_nftables_chain(rule, 'inet vrf_zones', chain, inverse=True) + # conntrack is only enabled once NAT, NAT66 or firewalling is enabled self.cli_set(['nat']) - self.cli_commit() + + for vrf in vrfs: + base = base_path + ['name', vrf] + self.cli_set(base + ['table', table]) + table = str(int(table) + 1) + # We need the commit inside the loop to trigger the bug in T6603 + self.cli_commit() # Conntrack rules should now be present for chain, rule in nftables_rules.items(): self.verify_nftables_chain(rule, 'inet vrf_zones', chain, inverse=False) + # T6603: there should be only ONE entry for the iifname/oifname in the chains + tmp = loads(cmd('sudo nft -j list table inet vrf_zones')) + num_rules = len(search("nftables[].rule[].chain", tmp)) + # ['vrf_zones_ct_in', 'vrf_zones_ct_out'] + self.assertEqual(num_rules, 2) + self.cli_delete(['nat']) if __name__ == '__main__': diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py index 184725573c3..33ef705595d 100755 --- a/src/conf_mode/vrf.py +++ b/src/conf_mode/vrf.py @@ -15,6 +15,7 @@ # along with this program. If not, see . from sys import exit +from jmespath import search from json import loads from vyos.config import Config @@ -70,6 +71,14 @@ def has_rule(af : str, priority : int, table : str=None): return True return False +def is_nft_vrf_zone_rule_setup() -> bool: + """ + Check if an nftables connection tracking rule already exists + """ + tmp = loads(cmd('sudo nft -j list table inet vrf_zones')) + num_rules = len(search("nftables[].rule[].chain", tmp)) + return bool(num_rules) + def vrf_interfaces(c, match): matched = [] old_level = c.get_level() @@ -302,7 +311,8 @@ def apply(vrf): nft_add_element = f'add element inet vrf_zones ct_iface_map {{ "{name}" : {table} }}' cmd(f'nft {nft_add_element}') - if vrf['conntrack']: + # Install nftables conntrack rules only once + if vrf['conntrack'] and not is_nft_vrf_zone_rule_setup(): for chain, rule in nftables_rules.items(): cmd(f'nft add rule inet vrf_zones {chain} {rule}') From 31acb42ecdf4ecf0f636f831f42a845b8a00d367 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Fri, 26 Jul 2024 13:43:31 +0200 Subject: [PATCH 06/64] vrf: T6603: improve code runtime when retrieving info from nftables vrf zone --- src/conf_mode/vrf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py index 33ef705595d..72b178c899f 100755 --- a/src/conf_mode/vrf.py +++ b/src/conf_mode/vrf.py @@ -273,6 +273,7 @@ def apply(vrf): if not has_rule(afi, 2000, 'l3mdev'): call(f'ip {afi} rule add pref 2000 l3mdev unreachable') + nft_vrf_zone_rule_setup = False for name, config in vrf['name'].items(): table = config['table'] if not interface_exists(name): @@ -311,8 +312,12 @@ def apply(vrf): nft_add_element = f'add element inet vrf_zones ct_iface_map {{ "{name}" : {table} }}' cmd(f'nft {nft_add_element}') + # Only call into nftables as long as there is nothing setup to avoid wasting + # CPU time and thus lenghten the commit process + if not nft_vrf_zone_rule_setup: + nft_vrf_zone_rule_setup = is_nft_vrf_zone_rule_setup() # Install nftables conntrack rules only once - if vrf['conntrack'] and not is_nft_vrf_zone_rule_setup(): + if vrf['conntrack'] and not nft_vrf_zone_rule_setup: for chain, rule in nftables_rules.items(): cmd(f'nft add rule inet vrf_zones {chain} {rule}') From 376e2d898f26c13a31f80d877f4e2621fd6efb0f Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Wed, 3 Jul 2024 23:14:45 -0700 Subject: [PATCH 07/64] T5873: vpn ipsec: re-write of ipsec updown hook --- python/vyos/ifconfig/vti.py | 19 +- python/vyos/utils/vti_updown_db.py | 194 +++++++++++++++++++ smoketest/scripts/cli/test_interfaces_vti.py | 3 +- smoketest/scripts/cli/test_vpn_ipsec.py | 19 +- src/conf_mode/vpn_ipsec.py | 54 +++++- src/etc/ipsec.d/vti-up-down | 53 ++--- 6 files changed, 302 insertions(+), 40 deletions(-) create mode 100644 python/vyos/utils/vti_updown_db.py diff --git a/python/vyos/ifconfig/vti.py b/python/vyos/ifconfig/vti.py index 9511386f44c..251cbeb3692 100644 --- a/python/vyos/ifconfig/vti.py +++ b/python/vyos/ifconfig/vti.py @@ -15,6 +15,7 @@ from vyos.ifconfig.interface import Interface from vyos.utils.dict import dict_search +from vyos.utils.vti_updown_db import vti_updown_db_exists, open_vti_updown_db_readonly @Interface.register class VTIIf(Interface): @@ -27,6 +28,10 @@ class VTIIf(Interface): }, } + def __init__(self, ifname, **kwargs): + self.bypass_vti_updown_db = kwargs.pop("bypass_vti_updown_db", False) + super().__init__(ifname, **kwargs) + def _create(self): # This table represents a mapping from VyOS internal config dict to # arguments used by iproute2. For more information please refer to: @@ -57,8 +62,18 @@ def _create(self): self.set_interface('admin_state', 'down') def set_admin_state(self, state): - """ Handled outside by /etc/ipsec.d/vti-up-down """ - pass + """ + Set interface administrative state to be 'up' or 'down'. + + The interface will only be brought 'up' if ith is attached to an + active ipsec site-to-site connection or remote access connection. + """ + if state == 'down' or self.bypass_vti_updown_db: + super().set_admin_state(state) + elif vti_updown_db_exists(): + with open_vti_updown_db_readonly() as db: + if db.wantsInterfaceUp(self.ifname): + super().set_admin_state(state) def get_mac(self): """ Get a synthetic MAC address. """ diff --git a/python/vyos/utils/vti_updown_db.py b/python/vyos/utils/vti_updown_db.py new file mode 100644 index 00000000000..b491fc6f2ea --- /dev/null +++ b/python/vyos/utils/vti_updown_db.py @@ -0,0 +1,194 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +import os + +from contextlib import contextmanager +from syslog import syslog + +VTI_WANT_UP_IFLIST = '/tmp/ipsec_vti_interfaces' + +def vti_updown_db_exists(): + """ Returns true if the database exists """ + return os.path.exists(VTI_WANT_UP_IFLIST) + +@contextmanager +def open_vti_updown_db_for_create_or_update(): + """ Opens the database for reading and writing, creating the database if it does not exist """ + if vti_updown_db_exists(): + f = open(VTI_WANT_UP_IFLIST, 'r+') + else: + f = open(VTI_WANT_UP_IFLIST, 'x+') + try: + db = VTIUpDownDB(f) + yield db + finally: + f.close() + +@contextmanager +def open_vti_updown_db_for_update(): + """ Opens the database for reading and writing, returning an error if it does not exist """ + f = open(VTI_WANT_UP_IFLIST, 'r+') + try: + db = VTIUpDownDB(f) + yield db + finally: + f.close() + +@contextmanager +def open_vti_updown_db_readonly(): + """ Opens the database for reading, returning an error if it does not exist """ + f = open(VTI_WANT_UP_IFLIST, 'r') + try: + db = VTIUpDownDB(f) + yield db + finally: + f.close() + +def remove_vti_updown_db(): + """ Brings down any interfaces referenced by the database and removes the database """ + # We need to process the DB first to bring down any interfaces still up + with open_vti_updown_db_for_update() as db: + db.removeAllOtherInterfaces([]) + # this usage of commit will only ever bring down interfaces, + # do not need to provide a functional interface dict supplier + db.commit(lambda _: None) + + os.unlink(VTI_WANT_UP_IFLIST) + +class VTIUpDownDB: + # The VTI Up-Down DB is a text-based database of space-separated "ifspecs". + # + # ifspecs can come in one of the two following formats: + # + # persistent format: + # indicates the named interface should always be up. + # + # connection format: :: + # indicates the named interface wants to be up due to an established + # connection using the protocol. + # + # The configuration tree and ipsec daemon connection up-down hook + # modify this file as needed and use it to determine when a + # particular event or configuration change should lead to changing + # the interface state. + + def __init__(self, f): + self._fileHandle = f + self._ifspecs = set([entry.strip() for entry in f.read().split(" ") if entry and not entry.isspace()]) + self._ifsUp = set() + self._ifsDown = set() + + def add(self, interface, connection = None, protocol = None): + """ + Adds a new entry to the DB. + + If an interface name, connection name, and protocol are supplied, + creates a connection entry. + + If only an interface name is specified, creates a persistent entry + for the given interface. + """ + ifspec = f"{interface}:{connection}:{protocol}" if (connection is not None and protocol is not None) else interface + if ifspec not in self._ifspecs: + self._ifspecs.add(ifspec) + self._ifsUp.add(interface) + self._ifsDown.discard(interface) + + def remove(self, interface, connection = None, protocol = None): + """ + Removes a matching entry from the DB. + + If no matching entry can be fonud, the operation returns successfully. + """ + ifspec = f"{interface}:{connection}:{protocol}" if (connection is not None and protocol is not None) else interface + if ifspec in self._ifspecs: + self._ifspecs.remove(ifspec) + interface_remains = False + for ifspec in self._ifspecs: + if ifspec.split(':')[0] == interface: + interface_remains = True + + if not interface_remains: + self._ifsDown.add(interface) + self._ifsUp.discard(interface) + + def wantsInterfaceUp(self, interface): + """ Returns whether the DB contains at least one entry referencing the given interface """ + for ifspec in self._ifspecs: + if ifspec.split(':')[0] == interface: + return True + + return False + + def removeAllOtherInterfaces(self, interface_list): + """ Removes all interfaces not included in the given list from the DB """ + updated_ifspecs = set([ifspec for ifspec in self._ifspecs if ifspec.split(':')[0] in interface_list]) + removed_ifspecs = self._ifspecs - updated_ifspecs + self._ifspecs = updated_ifspecs + interfaces_to_bring_down = [ifspec.split(':')[0] for ifspec in removed_ifspecs] + self._ifsDown.update(interfaces_to_bring_down) + self._ifsUp.difference_update(interfaces_to_bring_down) + + def setPersistentInterfaces(self, interface_list): + """ Updates the set of persistently up interfaces to match the given list """ + new_presistent_interfaces = set(interface_list) + current_presistent_interfaces = set([ifspec for ifspec in self._ifspecs if ':' not in ifspec]) + added_presistent_interfaces = new_presistent_interfaces - current_presistent_interfaces + removed_presistent_interfaces = current_presistent_interfaces - new_presistent_interfaces + + for interface in added_presistent_interfaces: + self.add(interface) + + for interface in removed_presistent_interfaces: + self.remove(interface) + + def commit(self, interface_dict_supplier): + """ + Writes the DB to disk and brings interfaces up and down as needed. + + Only interfaces referenced by entries modified in this DB session + are manipulated. If an interface is called to be brought up, the + provided interface_config_supplier function is invoked and expected + to return the config dictionary for the interface. + """ + from vyos.ifconfig import VTIIf + from vyos.utils.process import call + from vyos.utils.network import get_interface_config + + self._fileHandle.seek(0) + self._fileHandle.write(' '.join(self._ifspecs)) + self._fileHandle.truncate() + + for interface in self._ifsDown: + vti_link = get_interface_config(interface) + vti_link_up = (vti_link['operstate'] != 'DOWN' if 'operstate' in vti_link else False) + if vti_link_up: + call(f'sudo ip link set {interface} down') + syslog(f'Interface {interface} is admin down ...') + + self._ifsDown.clear() + + for interface in self._ifsUp: + vti_link = get_interface_config(interface) + vti_link_up = (vti_link['operstate'] != 'DOWN' if 'operstate' in vti_link else False) + if not vti_link_up: + vti = interface_dict_supplier(interface) + if 'disable' not in vti: + tmp = VTIIf(interface, bypass_vti_updown_db = True) + tmp.update(vti) + syslog(f'Interface {interface} is admin up ...') + + self._ifsUp.clear() diff --git a/smoketest/scripts/cli/test_interfaces_vti.py b/smoketest/scripts/cli/test_interfaces_vti.py index 871ac650bd6..8d90ca5ad26 100755 --- a/smoketest/scripts/cli/test_interfaces_vti.py +++ b/smoketest/scripts/cli/test_interfaces_vti.py @@ -39,7 +39,8 @@ def test_add_single_ip_address(self): self.cli_commit() - # VTI interface are always down and only brought up by IPSec + # VTI interfaces are default down and only brought up when an + # IPSec connection is configured to use them for intf in self._interfaces: self.assertTrue(is_intf_addr_assigned(intf, addr)) self.assertEqual(Interface(intf).get_admin_state(), 'down') diff --git a/smoketest/scripts/cli/test_vpn_ipsec.py b/smoketest/scripts/cli/test_vpn_ipsec.py index 2674b37b66a..3b8687b93e9 100755 --- a/smoketest/scripts/cli/test_vpn_ipsec.py +++ b/smoketest/scripts/cli/test_vpn_ipsec.py @@ -20,6 +20,7 @@ from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError +from vyos.ifconfig import Interface from vyos.utils.process import process_named_running from vyos.utils.file import read_file @@ -140,6 +141,7 @@ def tearDown(self): self.cli_delete(base_path) self.cli_delete(tunnel_path) + self.cli_delete(vti_path) self.cli_commit() # Check for no longer running process @@ -342,6 +344,12 @@ def test_site_to_site_vti(self): for line in swanctl_secrets_lines: self.assertRegex(swanctl_conf, fr'{line}') + # Site-to-site interfaces should start out as 'down' + self.assertEqual(Interface(vti).get_admin_state(), 'down') + + # Disable PKI + self.tearDownPKI() + def test_dmvpn(self): tunnel_if = 'tun100' @@ -478,9 +486,6 @@ def test_site_to_site_x509(self): self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{int_ca_name}.pem'))) self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) - # There is only one VTI test so no need to delete this globally in tearDown() - self.cli_delete(vti_path) - # Disable PKI self.tearDownPKI() @@ -1340,6 +1345,14 @@ def test_remote_access_vti(self): self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{ca_name}.pem'))) self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) + # Remote access interfaces should be set to 'up' during configure + self.assertEqual(Interface(vti).get_admin_state(), 'up') + + # Delete the connection to verify the VTI interfaces is taken down + self.cli_delete(base_path + ['remote-access', 'connection', conn_name]) + self.cli_commit() + self.assertEqual(Interface(vti).get_admin_state(), 'down') + self.tearDownPKI() if __name__ == '__main__': diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 2c1ddc245f1..789d37a77b5 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -30,6 +30,7 @@ from vyos.config import config_dict_merge from vyos.configdep import set_dependents from vyos.configdep import call_dependents +from vyos.configdict import get_interface_dict from vyos.configdict import leaf_node_changed from vyos.configverify import verify_interface_exists from vyos.configverify import dynamic_interface_pattern @@ -50,6 +51,9 @@ from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.process import call +from vyos.utils.vti_updown_db import vti_updown_db_exists +from vyos.utils.vti_updown_db import open_vti_updown_db_for_create_or_update +from vyos.utils.vti_updown_db import remove_vti_updown_db from vyos import ConfigError from vyos import airbag airbag.enable() @@ -107,6 +111,8 @@ def get_config(config=None): ipsec = config_dict_merge(default_values, ipsec) ipsec['dhcp_interfaces'] = set() + ipsec['enabled_vti_interfaces'] = set() + ipsec['persistent_vti_interfaces'] = set() ipsec['dhcp_no_address'] = {} ipsec['install_routes'] = 'no' if conf.exists(base + ["options", "disable-route-autoinstall"]) else default_install_routes ipsec['interface_change'] = leaf_node_changed(conf, base + ['interface']) @@ -124,6 +130,28 @@ def get_config(config=None): ipsec['l2tp_ike_default'] = 'aes256-sha1-modp1024,3des-sha1-modp1024' ipsec['l2tp_esp_default'] = 'aes256-sha1,3des-sha1' + # Collect the interface dicts for any refernced VTI interfaces in + # case we need to bring the interface up + ipsec['vti_interface_dicts'] = {} + + if 'site_to_site' in ipsec and 'peer' in ipsec['site_to_site']: + for peer, peer_conf in ipsec['site_to_site']['peer'].items(): + if 'vti' in peer_conf: + if 'bind' in peer_conf['vti']: + vti_interface = peer_conf['vti']['bind'] + if vti_interface not in ipsec['vti_interface_dicts']: + _, vti = get_interface_dict(conf, ['interfaces', 'vti'], vti_interface) + ipsec['vti_interface_dicts'][vti_interface] = vti + + if 'remote_access' in ipsec: + if 'connection' in ipsec['remote_access']: + for name, ra_conf in ipsec['remote_access']['connection'].items(): + if 'bind' in ra_conf: + vti_interface = ra_conf['bind'] + if vti_interface not in ipsec['vti_interface_dicts']: + _, vti = get_interface_dict(conf, ['interfaces', 'vti'], vti_interface) + ipsec['vti_interface_dicts'][vti_interface] = vti + return ipsec def get_dhcp_address(iface): @@ -308,13 +336,14 @@ def verify(ipsec): raise ConfigError('RADIUS authentication requires at least one server') if 'bind' in ra_conf: - if dict_search('options.disable_route_autoinstall', ipsec) == None: - Warning('It\'s recommended to use ipsec vti with the next command\n[set vpn ipsec option disable-route-autoinstall]') - vti_interface = ra_conf['bind'] - if not os.path.exists(f'/sys/class/net/{vti_interface}'): + if not interface_exists(vti_interface): raise ConfigError(f'VTI interface {vti_interface} for remote-access connection {name} does not exist!') + ipsec['enabled_vti_interfaces'].add(vti_interface) + # remote access VPN interfaces are always up regardless of whether clients are connected + ipsec['persistent_vti_interfaces'].add(vti_interface) + if 'pool' in ra_conf: if {'dhcp', 'radius'} <= set(ra_conf['pool']): raise ConfigError(f'Can not use both DHCP and RADIUS for address allocation '\ @@ -496,14 +525,11 @@ def verify(ipsec): if 'local_address' in peer_conf and 'dhcp_interface' in peer_conf: raise ConfigError(f"A single local-address or dhcp-interface is required when using VTI on site-to-site peer {peer}") - if dict_search('options.disable_route_autoinstall', - ipsec) == None: - Warning('It\'s recommended to use ipsec vti with the next command\n[set vpn ipsec option disable-route-autoinstall]') - if 'bind' in peer_conf['vti']: vti_interface = peer_conf['vti']['bind'] if not interface_exists(vti_interface): raise ConfigError(f'VTI interface {vti_interface} for site-to-site peer {peer} does not exist!') + ipsec['enabled_vti_interfaces'].add(vti_interface) if 'vti' not in peer_conf and 'tunnel' not in peer_conf: raise ConfigError(f"No VTI or tunnel specified on site-to-site peer {peer}") @@ -681,9 +707,21 @@ def apply(ipsec): systemd_service = 'strongswan.service' if not ipsec: call(f'systemctl stop {systemd_service}') + + if vti_updown_db_exists(): + remove_vti_updown_db() + else: call(f'systemctl reload-or-restart {systemd_service}') + if ipsec['enabled_vti_interfaces']: + with open_vti_updown_db_for_create_or_update() as db: + db.removeAllOtherInterfaces(ipsec['enabled_vti_interfaces']) + db.setPersistentInterfaces(ipsec['persistent_vti_interfaces']) + db.commit(lambda interface: ipsec['vti_interface_dicts'][interface]) + elif vti_updown_db_exists(): + remove_vti_updown_db() + if ipsec.get('nhrp_exists', False): try: call_dependents() diff --git a/src/etc/ipsec.d/vti-up-down b/src/etc/ipsec.d/vti-up-down index 01e9543c986..e1765ae857a 100755 --- a/src/etc/ipsec.d/vti-up-down +++ b/src/etc/ipsec.d/vti-up-down @@ -27,40 +27,41 @@ from syslog import LOG_INFO from vyos.configquery import ConfigTreeQuery from vyos.configdict import get_interface_dict -from vyos.ifconfig import VTIIf +from vyos.utils.commit import wait_for_commit_lock from vyos.utils.process import call -from vyos.utils.network import get_interface_config +from vyos.utils.vti_updown_db import open_vti_updown_db_for_update + +def supply_interface_dict(interface): + # Lazy-load the running config on first invocation + try: + conf = supply_interface_dict.cached_config + except AttributeError: + conf = ConfigTreeQuery() + supply_interface_dict.cached_config = conf + + _, vti = get_interface_dict(conf.config, ['interfaces', 'vti'], interface) + return vti if __name__ == '__main__': verb = os.getenv('PLUTO_VERB') connection = os.getenv('PLUTO_CONNECTION') interface = sys.argv[1] + if verb.endswith('-v6'): + protocol = 'v6' + else: + protocol = 'v4' + openlog(ident=f'vti-up-down', logoption=LOG_PID, facility=LOG_INFO) syslog(f'Interface {interface} {verb} {connection}') - if verb in ['up-client', 'up-host']: - call('sudo ip route delete default table 220') - - vti_link = get_interface_config(interface) - - if not vti_link: - syslog(f'Interface {interface} not found') - sys.exit(0) - - vti_link_up = (vti_link['operstate'] != 'DOWN' if 'operstate' in vti_link else False) + wait_for_commit_lock() - if verb in ['up-client', 'up-host']: - if not vti_link_up: - conf = ConfigTreeQuery() - _, vti = get_interface_dict(conf.config, ['interfaces', 'vti'], interface) - if 'disable' not in vti: - tmp = VTIIf(interface) - tmp.update(vti) - call(f'sudo ip link set {interface} up') - else: - call(f'sudo ip link set {interface} down') - syslog(f'Interface {interface} is admin down ...') - elif verb in ['down-client', 'down-host']: - if vti_link_up: - call(f'sudo ip link set {interface} down') + if verb in ['up-client', 'up-client-v6', 'up-host', 'up-host-v6']: + with open_vti_updown_db_for_update() as db: + db.add(interface, connection, protocol) + db.commit(supply_interface_dict) + elif verb in ['down-client', 'down-client-v6', 'down-host', 'down-host-v6']: + with open_vti_updown_db_for_update() as db: + db.remove(interface, connection, protocol) + db.commit(supply_interface_dict) From 404b641121d3f5f7686b6ad75236ff64b0733cf9 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Sun, 7 Jul 2024 03:11:00 -0700 Subject: [PATCH 08/64] T5873: vpn ipsec: ignore dhcp/vti settings when connection disabled --- src/conf_mode/vpn_ipsec.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 789d37a77b5..e8a0bc41473 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -280,7 +280,8 @@ def verify(ipsec): if not os.path.exists(f'{dhcp_base}/dhclient_{dhcp_interface}.conf'): raise ConfigError(f"Invalid dhcp-interface on remote-access connection {name}") - ipsec['dhcp_interfaces'].add(dhcp_interface) + if 'disable' not in ra_conf: + ipsec['dhcp_interfaces'].add(dhcp_interface) address = get_dhcp_address(dhcp_interface) count = 0 @@ -340,9 +341,10 @@ def verify(ipsec): if not interface_exists(vti_interface): raise ConfigError(f'VTI interface {vti_interface} for remote-access connection {name} does not exist!') - ipsec['enabled_vti_interfaces'].add(vti_interface) - # remote access VPN interfaces are always up regardless of whether clients are connected - ipsec['persistent_vti_interfaces'].add(vti_interface) + if 'disable' not in ra_conf: + ipsec['enabled_vti_interfaces'].add(vti_interface) + # remote access VPN interfaces are always up regardless of whether clients are connected + ipsec['persistent_vti_interfaces'].add(vti_interface) if 'pool' in ra_conf: if {'dhcp', 'radius'} <= set(ra_conf['pool']): @@ -507,7 +509,8 @@ def verify(ipsec): if not os.path.exists(f'{dhcp_base}/dhclient_{dhcp_interface}.conf'): raise ConfigError(f"Invalid dhcp-interface on site-to-site peer {peer}") - ipsec['dhcp_interfaces'].add(dhcp_interface) + if 'disable' not in peer_conf: + ipsec['dhcp_interfaces'].add(dhcp_interface) address = get_dhcp_address(dhcp_interface) count = 0 @@ -529,7 +532,8 @@ def verify(ipsec): vti_interface = peer_conf['vti']['bind'] if not interface_exists(vti_interface): raise ConfigError(f'VTI interface {vti_interface} for site-to-site peer {peer} does not exist!') - ipsec['enabled_vti_interfaces'].add(vti_interface) + if 'disable' not in peer_conf: + ipsec['enabled_vti_interfaces'].add(vti_interface) if 'vti' not in peer_conf and 'tunnel' not in peer_conf: raise ConfigError(f"No VTI or tunnel specified on site-to-site peer {peer}") From 50cf1746d3ab5e3666a3e502c67d7d853ae7f932 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Sun, 7 Jul 2024 03:19:02 -0700 Subject: [PATCH 09/64] T5873: vpn ipsec remote-access: improve child ESP session naming --- data/templates/ipsec/swanctl/remote_access.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/ipsec/swanctl/remote_access.j2 b/data/templates/ipsec/swanctl/remote_access.j2 index a3b61f781ac..c79f292b41f 100644 --- a/data/templates/ipsec/swanctl/remote_access.j2 +++ b/data/templates/ipsec/swanctl/remote_access.j2 @@ -46,7 +46,7 @@ {% endif %} } children { - ikev2-vpn { + {{ name }}-client { esp_proposals = {{ esp | get_esp_ike_cipher(ike) | join(',') }} {% if esp.life_bytes is vyos_defined %} life_bytes = {{ esp.life_bytes }} From 13957a9a25540bc1efd3e7930b08ec5a962580cd Mon Sep 17 00:00:00 2001 From: Indrajit Raychaudhuri Date: Mon, 29 Jul 2024 17:28:33 +0530 Subject: [PATCH 10/64] T6349: Fix typo in file name --- .github/workflows/{chceck-pr-message.yml => check-pr-message.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{chceck-pr-message.yml => check-pr-message.yml} (100%) diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/check-pr-message.yml similarity index 100% rename from .github/workflows/chceck-pr-message.yml rename to .github/workflows/check-pr-message.yml From 3d42009c0e3cf5ea7ea0ed167b4d8f655667edd8 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Tue, 30 Jul 2024 01:05:21 +1000 Subject: [PATCH 11/64] firewall: T4694: incomplete node checks in migration script This patch on #3616 will only attempt to fix ipsec matches in rules if the firewall config tree passed to migrate_chain() has rules attached. --- src/migration-scripts/firewall/16-to-17 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/migration-scripts/firewall/16-to-17 b/src/migration-scripts/firewall/16-to-17 index 9ad7a30f822..ad0706f04c9 100755 --- a/src/migration-scripts/firewall/16-to-17 +++ b/src/migration-scripts/firewall/16-to-17 @@ -27,13 +27,14 @@ # (nftables rejects 'meta ipsec' in output hooks), they are not considered here. # -import sys - from vyos.configtree import ConfigTree firewall_base = ['firewall'] def migrate_chain(config: ConfigTree, path: list[str]) -> None: + if not config.exists(path + ['rule']): + return + for rule_num in config.list_nodes(path + ['rule']): tmp_path = path + ['rule', rule_num, 'ipsec'] if config.exists(tmp_path + ['match-ipsec']): @@ -56,5 +57,4 @@ def migrate(config: ConfigTree) -> None: for base_hook in [['forward', 'filter'], ['input', 'filter'], ['prerouting', 'raw']]: tmp_path = firewall_base + [family] + base_hook - if config.exists(tmp_path): - migrate_chain(config, tmp_path) + migrate_chain(config, tmp_path) From 30111fa493c72e7182854f295bc88d9eecccf419 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 29 Jul 2024 19:16:43 +0100 Subject: [PATCH 12/64] vyos.configtree: T6620: allow list_nodes() to work on non-existent paths and return an empty list in that case (handy for migration scripts and the like) --- python/vyos/configtree.py | 9 ++++++--- src/migration-scripts/openvpn/1-to-2 | 8 ++------ src/migration-scripts/openvpn/2-to-3 | 8 ++------ src/migration-scripts/openvpn/3-to-4 | 6 +----- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 5775070e2cb..bd77ab8994c 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -1,5 +1,5 @@ # configtree -- a standalone VyOS config file manipulation library (Python bindings) -# Copyright (C) 2018-2022 VyOS maintainers and contributors +# Copyright (C) 2018-2024 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -290,7 +290,7 @@ def exists(self, path): else: return True - def list_nodes(self, path): + def list_nodes(self, path, path_must_exist=True): check_path(path) path_str = " ".join(map(str, path)).encode() @@ -298,7 +298,10 @@ def list_nodes(self, path): res = json.loads(res_json) if res is None: - raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + if path_must_exist: + raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + else: + return [] else: return res diff --git a/src/migration-scripts/openvpn/1-to-2 b/src/migration-scripts/openvpn/1-to-2 index b7b7d4c77d6..2baa7302ceb 100644 --- a/src/migration-scripts/openvpn/1-to-2 +++ b/src/migration-scripts/openvpn/1-to-2 @@ -20,12 +20,8 @@ from vyos.configtree import ConfigTree def migrate(config: ConfigTree) -> None: - if not config.exists(['interfaces', 'openvpn']): - # Nothing to do - return - - ovpn_intfs = config.list_nodes(['interfaces', 'openvpn']) - for i in ovpn_intfs: + ovpn_intfs = config.list_nodes(['interfaces', 'openvpn'], path_must_exist=False) + for i in ovpn_intfs: # Remove 'encryption cipher' and add this value to 'encryption ncp-ciphers' # for server and client mode. # Site-to-site mode still can use --cipher option diff --git a/src/migration-scripts/openvpn/2-to-3 b/src/migration-scripts/openvpn/2-to-3 index 0b9073ae66f..4e6b3c8b776 100644 --- a/src/migration-scripts/openvpn/2-to-3 +++ b/src/migration-scripts/openvpn/2-to-3 @@ -20,12 +20,8 @@ from vyos.configtree import ConfigTree def migrate(config: ConfigTree) -> None: - if not config.exists(['interfaces', 'openvpn']): - # Nothing to do - return - - ovpn_intfs = config.list_nodes(['interfaces', 'openvpn']) - for i in ovpn_intfs: + ovpn_intfs = config.list_nodes(['interfaces', 'openvpn'], path_must_exist=False) + for i in ovpn_intfs: mode = config.return_value(['interfaces', 'openvpn', i, 'mode']) if mode != 'server': # If it's a client or a site-to-site OpenVPN interface, diff --git a/src/migration-scripts/openvpn/3-to-4 b/src/migration-scripts/openvpn/3-to-4 index d3c76c7d332..0529491c130 100644 --- a/src/migration-scripts/openvpn/3-to-4 +++ b/src/migration-scripts/openvpn/3-to-4 @@ -18,11 +18,7 @@ from vyos.configtree import ConfigTree def migrate(config: ConfigTree) -> None: - if not config.exists(['interfaces', 'openvpn']): - # Nothing to do - return - - ovpn_intfs = config.list_nodes(['interfaces', 'openvpn']) + ovpn_intfs = config.list_nodes(['interfaces', 'openvpn'], path_must_exist=False) for i in ovpn_intfs: #Rename 'encryption ncp-ciphers' with 'encryption data-ciphers' ncp_cipher_path = ['interfaces', 'openvpn', i, 'encryption', 'ncp-ciphers'] From 6dfa557fe08ba746daed526a860b728331c74337 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 29 Jul 2024 22:32:17 +0200 Subject: [PATCH 13/64] T6560: action must be run on forked repo n order to properly build and test the code that is to be "merged in", we need to run this action on the source branch of the PR (pull_request) and not the target branch of the PR (pull_request_target) --- .github/workflows/package-smoketest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-smoketest.yml b/.github/workflows/package-smoketest.yml index 824cd64b1b3..0a8208b8718 100644 --- a/.github/workflows/package-smoketest.yml +++ b/.github/workflows/package-smoketest.yml @@ -1,7 +1,7 @@ name: VyOS ISO integration Test on: - pull_request_target: + pull_request: branches: - current paths: From adeac78ed6585b16102bd82581b54c75819714b2 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Tue, 30 Jul 2024 13:48:18 +1000 Subject: [PATCH 14/64] pbr: T6430: Allow forwarding into VRFs by name as well as route table IDs * PBR can only target table IDs up to 200 and the previous PR to extend the range was rejected * PBR with this PR can now also target VRFs directly by name, working around targeting problems for VRF table IDs outside the overlapping 100-200 range * Validation ensures rules can't target both a table ID and a VRF name (internally they are handled the same) * Added a simple accessor (get_vrf_table_id) for runtime mapping a VRF name to table ID, based on vyos.ifconfig.interface._set_vrf_ct_zone(). It does not replace that usage, as it deliberately does not handle non-VRF interface lookups (would fail with a KeyError). * Added route table ID lookup dict, global route table and VRF table defs to vyos.defaults. Table ID references have been updated in code touched by this PR. * Added a simple smoketest to validate 'set vrf' usage in PBR rules --- .../include/policy/route-common.xml.i | 18 +++++++ python/vyos/defaults.py | 10 ++++ python/vyos/firewall.py | 14 +++++- python/vyos/utils/network.py | 3 ++ smoketest/scripts/cli/test_policy_route.py | 49 +++++++++++++++++++ src/conf_mode/policy_route.py | 29 ++++++++--- 6 files changed, 116 insertions(+), 7 deletions(-) diff --git a/interface-definitions/include/policy/route-common.xml.i b/interface-definitions/include/policy/route-common.xml.i index 97795601eed..203be73e759 100644 --- a/interface-definitions/include/policy/route-common.xml.i +++ b/interface-definitions/include/policy/route-common.xml.i @@ -128,6 +128,24 @@ + + + VRF to forward packet with + + txt + VRF instance name + + + default + Forward into default global VRF + + + default + vrf name + + #include + + TCP Maximum Segment Size diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 9ccd925ce14..25ee453914a 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -50,3 +50,13 @@ component_version_json = os.path.join(directories['data'], 'component-versions.json') config_default = os.path.join(directories['data'], 'config.boot.default') + +rt_symbolic_names = { + # Standard routing tables for Linux & reserved IDs for VyOS + 'default': 253, # Confusingly, a final fallthru, not the default. + 'main': 254, # The actual global table used by iproute2 unless told otherwise. + 'local': 255, # Special kernel loopback table. +} + +rt_global_vrf = rt_symbolic_names['main'] +rt_global_table = rt_symbolic_names['main'] diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 40399f48114..89cd6818364 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -30,6 +30,9 @@ from vyos.utils.dict import dict_search_recursive from vyos.utils.process import cmd from vyos.utils.process import run +from vyos.utils.network import get_vrf_table_id +from vyos.defaults import rt_global_table +from vyos.defaults import rt_global_vrf # Conntrack def conntrack_required(conf): @@ -473,11 +476,20 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if 'mark' in rule_conf['set']: mark = rule_conf['set']['mark'] output.append(f'meta mark set {mark}') + if 'vrf' in rule_conf['set']: + set_table = True + vrf_name = rule_conf['set']['vrf'] + if vrf_name == 'default': + table = rt_global_vrf + else: + # NOTE: VRF->table ID lookup depends on the VRF iface already existing. + table = get_vrf_table_id(vrf_name) if 'table' in rule_conf['set']: set_table = True table = rule_conf['set']['table'] if table == 'main': - table = '254' + table = rt_global_table + if set_table: mark = 0x7FFFFFFF - int(table) output.append(f'meta mark set {mark}') if 'tcp_mss' in rule_conf['set']: diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index 8fce08de095..d297a1ddbe0 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -74,6 +74,9 @@ def get_vrf_members(vrf: str) -> list: pass return interfaces +def get_vrf_table_id(vrf: str): + return get_interface_config(vrf)['linkinfo']['info_data']['table'] + def get_interface_vrf(interface): """ Returns VRF of given interface """ from vyos.utils.dict import dict_search diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py index 462fc24d0d5..797ab97704c 100755 --- a/smoketest/scripts/cli/test_policy_route.py +++ b/smoketest/scripts/cli/test_policy_route.py @@ -25,6 +25,8 @@ conn_mark_set = '111' table_mark_offset = 0x7fffffff table_id = '101' +vrf = 'PBRVRF' +vrf_table_id = '102' interface = 'eth0' interface_wc = 'ppp*' interface_ip = '172.16.10.1/24' @@ -39,11 +41,14 @@ def setUpClass(cls): cls.cli_set(cls, ['interfaces', 'ethernet', interface, 'address', interface_ip]) cls.cli_set(cls, ['protocols', 'static', 'table', table_id, 'route', '0.0.0.0/0', 'interface', interface]) + + cls.cli_set(cls, ['vrf', 'name', vrf, 'table', vrf_table_id]) @classmethod def tearDownClass(cls): cls.cli_delete(cls, ['interfaces', 'ethernet', interface, 'address', interface_ip]) cls.cli_delete(cls, ['protocols', 'static', 'table', table_id]) + cls.cli_delete(cls, ['vrf', 'name', vrf]) super(TestPolicyRoute, cls).tearDownClass() @@ -180,6 +185,50 @@ def test_pbr_table(self): self.verify_rules(ip_rule_search) + def test_pbr_vrf(self): + self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'tcp']) + self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'port', '8888']) + self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'tcp', 'flags', 'syn']) + self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'tcp', 'flags', 'not', 'ack']) + self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'vrf', vrf]) + self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'protocol', 'tcp_udp']) + self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'destination', 'port', '8888']) + self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'set', 'vrf', vrf]) + + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) + + self.cli_commit() + + mark_hex = "{0:#010x}".format(table_mark_offset - int(vrf_table_id)) + + # IPv4 + + nftables_search = [ + [f'iifname "{interface}"', 'jump VYOS_PBR_UD_smoketest'], + ['tcp flags syn / syn,ack', 'tcp dport 8888', 'meta mark set ' + mark_hex] + ] + + self.verify_nftables(nftables_search, 'ip vyos_mangle') + + # IPv6 + + nftables6_search = [ + [f'iifname "{interface}"', 'jump VYOS_PBR6_UD_smoketest'], + ['meta l4proto { tcp, udp }', 'th dport 8888', 'meta mark set ' + mark_hex] + ] + + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') + + # IP rule fwmark -> table + + ip_rule_search = [ + ['fwmark ' + hex(table_mark_offset - int(vrf_table_id)), 'lookup ' + vrf] + ] + + self.verify_rules(ip_rule_search) + + def test_pbr_matching_criteria(self): self.cli_set(['policy', 'route', 'smoketest', 'default-log']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'udp']) diff --git a/src/conf_mode/policy_route.py b/src/conf_mode/policy_route.py index c58fe1bce72..da9002b593a 100755 --- a/src/conf_mode/policy_route.py +++ b/src/conf_mode/policy_route.py @@ -25,6 +25,9 @@ from vyos.utils.dict import dict_search_args from vyos.utils.process import cmd from vyos.utils.process import run +from vyos.utils.network import get_vrf_table_id +from vyos.defaults import rt_global_table +from vyos.defaults import rt_global_vrf from vyos import ConfigError from vyos import airbag airbag.enable() @@ -83,6 +86,9 @@ def verify_rule(policy, name, rule_conf, ipv6, rule_id): if not tcp_flags or 'syn' not in tcp_flags: raise ConfigError(f'{name} rule {rule_id}: TCP SYN flag must be set to modify TCP-MSS') + if 'vrf' in rule_conf['set'] and 'table' in rule_conf['set']: + raise ConfigError(f'{name} rule {rule_id}: Cannot set both forwarding route table and VRF') + tcp_flags = dict_search_args(rule_conf, 'tcp', 'flags') if tcp_flags: if dict_search_args(rule_conf, 'protocol') != 'tcp': @@ -152,15 +158,26 @@ def apply_table_marks(policy): for name, pol_conf in policy[route].items(): if 'rule' in pol_conf: for rule_id, rule_conf in pol_conf['rule'].items(): + vrf_table_id = None set_table = dict_search_args(rule_conf, 'set', 'table') - if set_table: + set_vrf = dict_search_args(rule_conf, 'set', 'vrf') + if set_vrf: + if set_vrf == 'default': + vrf_table_id = rt_global_vrf + else: + vrf_table_id = get_vrf_table_id(set_vrf) + elif set_table: if set_table == 'main': - set_table = '254' - if set_table in tables: + vrf_table_id = rt_global_table + else: + vrf_table_id = set_table + if vrf_table_id is not None: + vrf_table_id = int(vrf_table_id) + if vrf_table_id in tables: continue - tables.append(set_table) - table_mark = mark_offset - int(set_table) - cmd(f'{cmd_str} rule add pref {set_table} fwmark {table_mark} table {set_table}') + tables.append(vrf_table_id) + table_mark = mark_offset - vrf_table_id + cmd(f'{cmd_str} rule add pref {vrf_table_id} fwmark {table_mark} table {vrf_table_id}') def cleanup_table_marks(): for cmd_str in ['ip', 'ip -6']: From 9b99a01653e3315b1abc9ef98824ca71bd283047 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 30 Jul 2024 08:07:29 +0200 Subject: [PATCH 15/64] pbr: T6430: refactor to use vyos.utils.network.get_vrf_tableid() Commit 452068ce78 ("interfaces: T6592: moving an interface between VRF instances failed") added a similar but more detailed implementation of get_vrf_table_id() that was added in commit adeac78ed of this PR. Move to the common available implementation. --- python/vyos/firewall.py | 6 +++--- python/vyos/utils/network.py | 3 --- src/conf_mode/policy_route.py | 6 +++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 89cd6818364..facd498ca10 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -30,7 +30,7 @@ from vyos.utils.dict import dict_search_recursive from vyos.utils.process import cmd from vyos.utils.process import run -from vyos.utils.network import get_vrf_table_id +from vyos.utils.network import get_vrf_tableid from vyos.defaults import rt_global_table from vyos.defaults import rt_global_vrf @@ -482,8 +482,8 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if vrf_name == 'default': table = rt_global_vrf else: - # NOTE: VRF->table ID lookup depends on the VRF iface already existing. - table = get_vrf_table_id(vrf_name) + # NOTE: VRF->table ID lookup depends on the VRF iface already existing. + table = get_vrf_tableid(vrf_name) if 'table' in rule_conf['set']: set_table = True table = rule_conf['set']['table'] diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index d297a1ddbe0..8fce08de095 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -74,9 +74,6 @@ def get_vrf_members(vrf: str) -> list: pass return interfaces -def get_vrf_table_id(vrf: str): - return get_interface_config(vrf)['linkinfo']['info_data']['table'] - def get_interface_vrf(interface): """ Returns VRF of given interface """ from vyos.utils.dict import dict_search diff --git a/src/conf_mode/policy_route.py b/src/conf_mode/policy_route.py index da9002b593a..223175b8ab4 100755 --- a/src/conf_mode/policy_route.py +++ b/src/conf_mode/policy_route.py @@ -25,7 +25,7 @@ from vyos.utils.dict import dict_search_args from vyos.utils.process import cmd from vyos.utils.process import run -from vyos.utils.network import get_vrf_table_id +from vyos.utils.network import get_vrf_tableid from vyos.defaults import rt_global_table from vyos.defaults import rt_global_vrf from vyos import ConfigError @@ -165,14 +165,14 @@ def apply_table_marks(policy): if set_vrf == 'default': vrf_table_id = rt_global_vrf else: - vrf_table_id = get_vrf_table_id(set_vrf) + vrf_table_id = get_vrf_tableid(set_vrf) elif set_table: if set_table == 'main': vrf_table_id = rt_global_table else: vrf_table_id = set_table if vrf_table_id is not None: - vrf_table_id = int(vrf_table_id) + vrf_table_id = int(vrf_table_id) if vrf_table_id in tables: continue tables.append(vrf_table_id) From e97d86e619e134f4dfda06efb7df4a3296d17b95 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Mon, 29 Jul 2024 23:22:05 -0700 Subject: [PATCH 16/64] T6617: T6618: vpn ipsec remote-access: fix profile generators --- data/templates/ipsec/ios_profile.j2 | 9 ++- data/templates/ipsec/windows_profile.j2 | 2 +- src/op_mode/ikev2_profile_generator.py | 85 +++++++++++++++++++++---- 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/data/templates/ipsec/ios_profile.j2 b/data/templates/ipsec/ios_profile.j2 index 935acbf8ee7..966fad43396 100644 --- a/data/templates/ipsec/ios_profile.j2 +++ b/data/templates/ipsec/ios_profile.j2 @@ -55,9 +55,11 @@ AuthenticationMethod Certificate +{% if authentication.client_mode.startswith("eap") %} ExtendedAuthEnabled 1 +{% endif %} IKESecurityAssociationParameters @@ -78,9 +80,14 @@ {{ esp_encryption.encryption }} IntegrityAlgorithm {{ esp_encryption.hash }} +{% if esp_encryption.pfs is vyos_defined %} DiffieHellmanGroup - {{ ike_encryption.dh_group }} + {{ esp_encryption.pfs }} +{% endif %} + + EnablePFS + {{ '1' if esp_encryption.pfs is vyos_defined else '0' }} {% if ca_certificates is vyos_defined %} diff --git a/data/templates/ipsec/windows_profile.j2 b/data/templates/ipsec/windows_profile.j2 index 8c26944be51..b5042f98757 100644 --- a/data/templates/ipsec/windows_profile.j2 +++ b/data/templates/ipsec/windows_profile.j2 @@ -1,4 +1,4 @@ Remove-VpnConnection -Name "{{ vpn_name }}" -Force -PassThru Add-VpnConnection -Name "{{ vpn_name }}" -ServerAddress "{{ remote }}" -TunnelType "Ikev2" -Set-VpnConnectionIPsecConfiguration -ConnectionName "{{ vpn_name }}" -AuthenticationTransformConstants {{ ike_encryption.encryption }} -CipherTransformConstants {{ ike_encryption.encryption }} -EncryptionMethod {{ esp_encryption.encryption }} -IntegrityCheckMethod {{ esp_encryption.hash }} -PfsGroup None -DHGroup "Group{{ ike_encryption.dh_group }}" -PassThru -Force +Set-VpnConnectionIPsecConfiguration -ConnectionName "{{ vpn_name }}" -AuthenticationTransformConstants {{ ike_encryption.encryption }} -CipherTransformConstants {{ ike_encryption.encryption }} -EncryptionMethod {{ esp_encryption.encryption }} -IntegrityCheckMethod {{ esp_encryption.hash }} -PfsGroup {{ esp_encryption.pfs }} -DHGroup {{ ike_encryption.dh_group }} -PassThru -Force diff --git a/src/op_mode/ikev2_profile_generator.py b/src/op_mode/ikev2_profile_generator.py index b193d810914..cf2bc6d5c1f 100755 --- a/src/op_mode/ikev2_profile_generator.py +++ b/src/op_mode/ikev2_profile_generator.py @@ -105,10 +105,39 @@ } # IOS 14.2 and later do no support dh-group 1,2 and 5. Supported DH groups would -# be: 14, 15, 16, 17, 18, 19, 20, 21, 31 -ios_supported_dh_groups = ['14', '15', '16', '17', '18', '19', '20', '21', '31'] -# Windows 10 only allows a limited set of DH groups -windows_supported_dh_groups = ['1', '2', '14', '24'] +# be: 14, 15, 16, 17, 18, 19, 20, 21, 31, 32 +vyos2apple_dh_group = { + '14' : '14', + '15' : '15', + '16' : '16', + '17' : '17', + '18' : '18', + '19' : '19', + '20' : '20', + '21' : '21', + '31' : '31', + '32' : '32' +} + +# Newer versions of Windows support groups 19 and 20, albeit under a different naming convention +vyos2windows_dh_group = { + '1' : 'Group1', + '2' : 'Group2', + '14' : 'Group14', + '19' : 'ECP256', + '20' : 'ECP384', + '24' : 'Group24' +} + +# For PFS, Windows also has its own inconsistent naming scheme for each group +vyos2windows_pfs_group = { + '1' : 'PFS1', + '2' : 'PFS2', + '14' : 'PFS2048', + '19' : 'ECP256', + '20' : 'ECP384', + '24' : 'PFS24' +} parser = argparse.ArgumentParser() parser.add_argument('--os', const='all', nargs='?', choices=['ios', 'windows'], help='Operating system used for config generation', required=True) @@ -181,7 +210,7 @@ # https://stackoverflow.com/a/9427216 data['ca_certificates'] = [dict(t) for t in {tuple(d.items()) for d in data['ca_certificates']}] -esp_proposals = conf.get_config_dict(ipsec_base + ['esp-group', data['esp_group'], 'proposal'], +esp_group = conf.get_config_dict(ipsec_base + ['esp-group', data['esp_group']], key_mangling=('-', '_'), get_first_key=True) ike_proposal = conf.get_config_dict(ipsec_base + ['ike-group', data['ike_group'], 'proposal'], key_mangling=('-', '_'), get_first_key=True) @@ -192,7 +221,29 @@ vyos2client_cipher = vyos2apple_cipher if args.os == 'ios' else vyos2windows_cipher; vyos2client_integrity = vyos2apple_integrity if args.os == 'ios' else vyos2windows_integrity; -supported_dh_groups = ios_supported_dh_groups if args.os == 'ios' else windows_supported_dh_groups; +vyos2client_dh_group = vyos2apple_dh_group if args.os == 'ios' else vyos2windows_dh_group + +def transform_pfs(pfs, ike_dh_group): + pfs_enabled = (pfs != 'disable') + if pfs == 'enable': + pfs_dh_group = ike_dh_group + elif pfs.startswith('dh-group'): + pfs_dh_group = pfs.removeprefix('dh-group') + + if args.os == 'ios': + if pfs_enabled: + if pfs_dh_group not in set(vyos2apple_dh_group): + exit(f'The PFS group configured for "{args.connection}" is not supported by the client!') + return pfs_dh_group + else: + return None + else: + if pfs_enabled: + if pfs_dh_group not in set(vyos2windows_pfs_group): + exit(f'The PFS group configured for "{args.connection}" is not supported by the client!') + return vyos2windows_pfs_group[ pfs_dh_group ] + else: + return 'None' # Create a dictionary containing client conform IKE settings ike = {} @@ -201,24 +252,28 @@ if {'dh_group', 'encryption', 'hash'} <= set(proposal): if (proposal['encryption'] in set(vyos2client_cipher) and proposal['hash'] in set(vyos2client_integrity) and - proposal['dh_group'] in set(supported_dh_groups)): + proposal['dh_group'] in set(vyos2client_dh_group)): # We 're-code' from the VyOS IPsec proposals to the Apple naming scheme proposal['encryption'] = vyos2client_cipher[ proposal['encryption'] ] proposal['hash'] = vyos2client_integrity[ proposal['hash'] ] + # DH group will need to be transformed later after we calculate PFS group ike.update( { str(count) : proposal } ) count += 1 -# Create a dictionary containing Apple conform ESP settings +# Create a dictionary containing client conform ESP settings esp = {} count = 1 -for _, proposal in esp_proposals.items(): +for _, proposal in esp_group['proposal'].items(): if {'encryption', 'hash'} <= set(proposal): if proposal['encryption'] in set(vyos2client_cipher) and proposal['hash'] in set(vyos2client_integrity): # We 're-code' from the VyOS IPsec proposals to the Apple naming scheme proposal['encryption'] = vyos2client_cipher[ proposal['encryption'] ] proposal['hash'] = vyos2client_integrity[ proposal['hash'] ] + # Copy PFS setting from the group, if present (we will need to + # transform this later once the IKE group is selected) + proposal['pfs'] = esp_group.get('pfs', 'enable') esp.update( { str(count) : proposal } ) count += 1 @@ -230,8 +285,10 @@ tmp += f'({number}) Encryption {options["encryption"]}, Integrity {options["hash"]}, DH group {options["dh_group"]}\n' tmp += '\nSelect one of the above IKE groups: ' data['ike_encryption'] = ike[ ask_input(tmp, valid_responses=list(ike)) ] - else: + elif len(ike) == 1: data['ike_encryption'] = ike['1'] + else: + exit(f'None of the configured IKE proposals for "{args.connection}" are supported by the client!') if len(esp) > 1: tmp = '\n' @@ -239,12 +296,18 @@ tmp += f'({number}) Encryption {options["encryption"]}, Integrity {options["hash"]}\n' tmp += '\nSelect one of the above ESP groups: ' data['esp_encryption'] = esp[ ask_input(tmp, valid_responses=list(esp)) ] - else: + elif len(esp) == 1: data['esp_encryption'] = esp['1'] + else: + exit(f'None of the configured ESP proposals for "{args.connection}" are supported by the client!') except KeyboardInterrupt: exit("Interrupted") +# Transform the DH and PFS groups now that all selections are known +data['esp_encryption']['pfs'] = transform_pfs(data['esp_encryption']['pfs'], data['ike_encryption']['dh_group']) +data['ike_encryption']['dh_group'] = vyos2client_dh_group[ data['ike_encryption']['dh_group'] ] + print('\n\n==== ====') if args.os == 'ios': print(render_to_string('ipsec/ios_profile.j2', data)) From a81d270b166dbe939167523ee0b837fe701d3594 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:41:57 +0530 Subject: [PATCH 17/64] T6572: trigger remote pr only for circinus pr merge (#3899) --- .github/workflows/trigger-pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/trigger-pr.yml b/.github/workflows/trigger-pr.yml index 0e28b460fc6..f88458a8128 100644 --- a/.github/workflows/trigger-pr.yml +++ b/.github/workflows/trigger-pr.yml @@ -5,13 +5,13 @@ on: types: - closed branches: - - current - + - circinus + jobs: trigger-PR: uses: vyos/.github/.github/workflows/trigger-pr.yml@current with: - source_branch: 'current' + source_branch: 'circinus' target_branch: 'circinus' secrets: REMOTE_REPO: ${{ secrets.REMOTE_REPO }} From bc9049ebd76576d727fa87b10b96d1616950237c Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Mon, 8 Jul 2024 23:58:25 +1000 Subject: [PATCH 18/64] system: op-mode: T3334: allow delayed getty restart when configuring serial ports * Created op-mode command "restart serial console" * Relocated service control to vyos.utils.serial helpers, used by conf- and op-mode serial console handling * Checking for logged-in serial sessions that may be affected by getty reconfig * Warning the user when changes are committed and serial sessions are active, otherwise restart services as normal. No prompts issued during commit, all config gen/commit steps still occur except for the service restarts (everything remains consistent) * To apply committed changes, user will need to run "restart serial console" to complete the process or reboot the whole router * Added additional flags and target filtering for generic use of helpers. --- op-mode-definitions/restart-serial.xml.in | 31 ++++++ python/vyos/utils/serial.py | 117 ++++++++++++++++++++++ src/completion/list_login_ttys.py | 25 +++++ src/conf_mode/system_console.py | 15 ++- src/op_mode/serial.py | 38 +++++++ 5 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 op-mode-definitions/restart-serial.xml.in create mode 100644 python/vyos/utils/serial.py create mode 100644 src/completion/list_login_ttys.py create mode 100644 src/op_mode/serial.py diff --git a/op-mode-definitions/restart-serial.xml.in b/op-mode-definitions/restart-serial.xml.in new file mode 100644 index 00000000000..4d8a03633d0 --- /dev/null +++ b/op-mode-definitions/restart-serial.xml.in @@ -0,0 +1,31 @@ + + + + + + + Restart services on serial ports + + + + + Restart serial console service for login TTYs + + sudo ${vyos_op_scripts_dir}/serial.py restart_console + + + + Restart specific TTY device + + + + + sudo ${vyos_op_scripts_dir}/serial.py restart_console --device-name "$5" + + + + + + + + diff --git a/python/vyos/utils/serial.py b/python/vyos/utils/serial.py new file mode 100644 index 00000000000..7a662bf96d6 --- /dev/null +++ b/python/vyos/utils/serial.py @@ -0,0 +1,117 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +import os, re, json +from typing import List + +from vyos.base import Warning +from vyos.utils.io import ask_yes_no +from vyos.utils.process import cmd + +GLOB_GETTY_UNITS = 'serial-getty@*.service' +RE_GETTY_DEVICES = re.compile(r'.+@(.+).service$') + +SD_UNIT_PATH = '/run/systemd/system' +UTMP_PATH = '/run/utmp' + +def get_serial_units(include_devices=[]): + # Since we cannot depend on the current config for decommissioned ports, + # we just grab everything that systemd knows about. + tmp = cmd(f'systemctl list-units {GLOB_GETTY_UNITS} --all --output json --no-pager') + getty_units = json.loads(tmp) + for sdunit in getty_units: + m = RE_GETTY_DEVICES.search(sdunit['unit']) + if m is None: + Warning(f'Serial console unit name "{sdunit["unit"]}" is malformed and cannot be checked for activity!') + continue + + getty_device = m.group(1) + if include_devices and getty_device not in include_devices: + continue + + sdunit['device'] = getty_device + + return getty_units + +def get_authenticated_ports(units): + connected = [] + ports = [ x['device'] for x in units if 'device' in x ] + # + # utmpdump just gives us an easily parseable dump of currently logged-in sessions, for eg: + # $ utmpdump /run/utmp + # Utmp dump of /run/utmp + # [2] [00000] [~~ ] [reboot ] [~ ] [6.6.31-amd64-vyos ] [0.0.0.0 ] [2024-06-18T13:56:53,958484+00:00] + # [1] [00051] [~~ ] [runlevel] [~ ] [6.6.31-amd64-vyos ] [0.0.0.0 ] [2024-06-18T13:57:01,790808+00:00] + # [6] [03178] [tty1] [LOGIN ] [tty1 ] [ ] [0.0.0.0 ] [2024-06-18T13:57:31,015392+00:00] + # [7] [37151] [ts/0] [vyos ] [pts/0 ] [10.9.8.7 ] [10.9.8.7 ] [2024-07-04T13:42:08,760892+00:00] + # [8] [24812] [ts/1] [ ] [pts/1 ] [10.9.8.7 ] [10.9.8.7 ] [2024-06-20T18:10:07,309365+00:00] + # + # We can safely skip blank or LOGIN sessions with valid device names. + # + for line in cmd(f'utmpdump {UTMP_PATH}').splitlines(): + row = line.split('] [') + user_name = row[3].strip() + user_term = row[4].strip() + if user_name and user_name != 'LOGIN' and user_term in ports: + connected.append(user_term) + + return connected + +def restart_login_consoles(prompt_user=False, quiet=True, devices: List[str]=[]): + # restart_login_consoles() is called from both conf- and op-mode scripts, including + # the warning messages and user prompts common to both. + # + # The default case, called with no arguments, is a simple serial-getty restart & + # cleanup wrapper with no output or prompts that can be used from anywhere. + # + # quiet and prompt_user args have been split from an original "no_prompt", in + # order to support the completely silent default use case. "no_prompt" would + # only suppress the user interactive prompt. + # + # quiet intentionally does not suppress a vyos.base.Warning() for malformed + # device names in _get_serial_units(). + # + cmd('systemctl daemon-reload') + + units = get_serial_units(devices) + connected = get_authenticated_ports(units) + + if connected: + if not quiet: + print('There are user sessions connected via serial console that will be terminated\n' \ + 'when serial console settings are changed.\n') # extra newline is deliberate. + if not prompt_user: + # This flag is used by conf_mode/system_console.py to reset things, if there's + # a problem, the user should issue a manual restart for serial-getty. + print('Please ensure all settings are committed and saved before issuing a\n' \ + '"restart serial console" command to apply new configuration.') + if not prompt_user: + return False + if not ask_yes_no('Any uncommitted changes from these sessions will be lost and in-progress actions\n' \ + 'may be left in an inconsistent state. Continue?'): + return False + + for unit in units: + if 'device' not in unit: + continue # malformed or filtered. + unit_name = unit['unit'] + unit_device = unit['device'] + if os.path.exists(os.path.join(SD_UNIT_PATH, unit_name)): + cmd(f'systemctl restart {unit_name}') + else: + # Deleted stubs don't need to be restarted, just shut them down. + cmd(f'systemctl stop {unit_name}') + + return True diff --git a/src/completion/list_login_ttys.py b/src/completion/list_login_ttys.py new file mode 100644 index 00000000000..4d77a1b8b4e --- /dev/null +++ b/src/completion/list_login_ttys.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from vyos.utils.serial import get_serial_units + +if __name__ == '__main__': + # Autocomplete uses runtime state rather than the config tree, as a manual + # restart/cleanup may be needed for deleted devices. + tty_completions = [ '' ] + [ x['device'] for x in get_serial_units() if 'device' in x ] + print(' '.join(tty_completions)) + + diff --git a/src/conf_mode/system_console.py b/src/conf_mode/system_console.py index 19bbb887580..27bf92e0b14 100755 --- a/src/conf_mode/system_console.py +++ b/src/conf_mode/system_console.py @@ -19,8 +19,10 @@ from vyos.config import Config from vyos.utils.process import call +from vyos.utils.serial import restart_login_consoles from vyos.system import grub_util from vyos.template import render +from vyos.defaults import directories from vyos import ConfigError from vyos import airbag airbag.enable() @@ -74,7 +76,6 @@ def generate(console): for root, dirs, files in os.walk(base_dir): for basename in files: if 'serial-getty' in basename: - call(f'systemctl stop {basename}') os.unlink(os.path.join(root, basename)) if not console or 'device' not in console: @@ -122,6 +123,11 @@ def apply(console): # Reload systemd manager configuration call('systemctl daemon-reload') + # Service control moved to vyos.utils.serial to unify checks and prompts. + # If users are connected, we want to show an informational message on completing + # the process, but not halt configuration processing with an interactive prompt. + restart_login_consoles(prompt_user=False, quiet=False) + if not console: return None @@ -129,13 +135,6 @@ def apply(console): # Configure screen blank powersaving on VGA console call('/usr/bin/setterm -blank 15 -powersave powerdown -powerdown 60 -term linux /dev/tty1 2>&1') - # Start getty process on configured serial interfaces - for device in console['device']: - # Only start console if it exists on the running system. If a user - # detaches a USB serial console and reboots - it should not fail! - if os.path.exists(f'/dev/{device}'): - call(f'systemctl restart serial-getty@{device}.service') - return None if __name__ == '__main__': diff --git a/src/op_mode/serial.py b/src/op_mode/serial.py new file mode 100644 index 00000000000..a5864872b3e --- /dev/null +++ b/src/op_mode/serial.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys, typing + +import vyos.opmode +from vyos.utils.serial import restart_login_consoles as _restart_login_consoles + +def restart_console(device_name: typing.Optional[str]): + # Service control moved to vyos.utils.serial to unify checks and prompts. + # If users are connected, we want to show an informational message and a prompt + # to continue, verifying that the user acknowledges possible interruptions. + if device_name: + _restart_login_consoles(prompt_user=True, quiet=False, devices=[device_name]) + else: + _restart_login_consoles(prompt_user=True, quiet=False) + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) From b3b31153963cc4338e8229f9f94b339682dd73a0 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 30 Jul 2024 15:50:45 +0200 Subject: [PATCH 19/64] system: op-mode: T3334: replace some print() statements with Warning() Make it more obvious for the user aber the severity of his action. --- python/vyos/utils/serial.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/vyos/utils/serial.py b/python/vyos/utils/serial.py index 7a662bf96d6..b646f881e07 100644 --- a/python/vyos/utils/serial.py +++ b/python/vyos/utils/serial.py @@ -90,17 +90,18 @@ def restart_login_consoles(prompt_user=False, quiet=True, devices: List[str]=[]) if connected: if not quiet: - print('There are user sessions connected via serial console that will be terminated\n' \ - 'when serial console settings are changed.\n') # extra newline is deliberate. + Warning('There are user sessions connected via serial console that '\ + 'will be terminated when serial console settings are changed!') if not prompt_user: # This flag is used by conf_mode/system_console.py to reset things, if there's - # a problem, the user should issue a manual restart for serial-getty. - print('Please ensure all settings are committed and saved before issuing a\n' \ - '"restart serial console" command to apply new configuration.') + # a problem, the user should issue a manual restart for serial-getty. + Warning('Please ensure all settings are committed and saved before issuing a ' \ + '"restart serial console" command to apply new configuration!') if not prompt_user: return False - if not ask_yes_no('Any uncommitted changes from these sessions will be lost and in-progress actions\n' \ - 'may be left in an inconsistent state. Continue?'): + if not ask_yes_no('Any uncommitted changes from these sessions will be lost\n' \ + 'and in-progress actions may be left in an inconsistent state.\n'\ + '\nContinue?'): return False for unit in units: From cb1834742f4ed01d99d6396af8339dd59788ef65 Mon Sep 17 00:00:00 2001 From: aapostoliuk <108394744+aapostoliuk@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:26:25 +0300 Subject: [PATCH 20/64] ipsec: T6148: Removed unused imports (#3915) Removed unused pprint module --- src/op_mode/ipsec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index c8f5072da37..02ba126b468 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -13,7 +13,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import pprint import re import sys import typing From ab331fab9e92a69e68080d413bf926db14ac354b Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 31 Jul 2024 15:25:47 +0000 Subject: [PATCH 21/64] T5657: Add VRF support for zabbix-agent To start the service under VRF requires starting under User=root otherwise it had issues with cgroups --- data/templates/zabbix-agent/10-override.conf.j2 | 5 ++++- interface-definitions/service_monitoring_zabbix-agent.xml.in | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/data/templates/zabbix-agent/10-override.conf.j2 b/data/templates/zabbix-agent/10-override.conf.j2 index 7c296e8fdac..f6bd6500d34 100644 --- a/data/templates/zabbix-agent/10-override.conf.j2 +++ b/data/templates/zabbix-agent/10-override.conf.j2 @@ -1,3 +1,4 @@ +{% set zabbix_command = 'ip vrf exec ' ~ vrf ~ ' ' if vrf is vyos_defined else '' %} [Unit] After= After=vyos-router.service @@ -5,9 +6,11 @@ ConditionPathExists= ConditionPathExists=/run/zabbix/zabbix-agent2.conf [Service] +User= +User=root EnvironmentFile= ExecStart= -ExecStart=/usr/sbin/zabbix_agent2 --config /run/zabbix/zabbix-agent2.conf --foreground +ExecStart={{ zabbix_command }}/usr/sbin/zabbix_agent2 --config /run/zabbix/zabbix-agent2.conf --foreground WorkingDirectory= WorkingDirectory=/run/zabbix Restart=always diff --git a/interface-definitions/service_monitoring_zabbix-agent.xml.in b/interface-definitions/service_monitoring_zabbix-agent.xml.in index 3754e914578..e44b31312db 100644 --- a/interface-definitions/service_monitoring_zabbix-agent.xml.in +++ b/interface-definitions/service_monitoring_zabbix-agent.xml.in @@ -185,6 +185,7 @@ 3 + #include From 4acad3eb8d9be173b76fecafc32b0c70eae9b192 Mon Sep 17 00:00:00 2001 From: fett0 Date: Wed, 31 Jul 2024 18:21:25 +0000 Subject: [PATCH 22/64] OPENVPN: T6555: add server-bridge options in mode server --- data/templates/openvpn/server.conf.j2 | 4 +- .../interfaces_openvpn.xml.in | 56 +++++++++++++++++++ .../scripts/cli/test_interfaces_openvpn.py | 55 ++++++++++++++++++ src/conf_mode/interfaces_openvpn.py | 16 ++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/data/templates/openvpn/server.conf.j2 b/data/templates/openvpn/server.conf.j2 index f6951969746..a8674ec3579 100644 --- a/data/templates/openvpn/server.conf.j2 +++ b/data/templates/openvpn/server.conf.j2 @@ -90,7 +90,9 @@ server-ipv6 {{ subnet }} {% endif %} {% endfor %} {% endif %} - +{% if server.server_bridge is vyos_defined and server.server_bridge.disable is not vyos_defined %} +server-bridge {{ server.server_bridge.gateway }} {{ server.server_bridge.subnet_mask }} {{ server.server_bridge.start }} {{ server.server_bridge.stop if server.server_bridge.stop is vyos_defined }} +{% endif %} {% if server.client_ip_pool is vyos_defined and server.client_ip_pool.disable is not vyos_defined %} ifconfig-pool {{ server.client_ip_pool.start }} {{ server.client_ip_pool.stop }} {{ server.client_ip_pool.subnet_mask if server.client_ip_pool.subnet_mask is vyos_defined }} {% endif %} diff --git a/interface-definitions/interfaces_openvpn.xml.in b/interface-definitions/interfaces_openvpn.xml.in index 13ef3ae5bc4..a988bf36c7c 100644 --- a/interface-definitions/interfaces_openvpn.xml.in +++ b/interface-definitions/interfaces_openvpn.xml.in @@ -445,6 +445,62 @@ + + + Used with TAP device (layer 2) + + + #include + + + First IP address in the pool + + + + + ipv4 + IPv4 address + + + + + + Last IP address in the pool + + + + + ipv4 + IPv4 address + + + + + + Subnet mask pushed to dynamic clients. + + + + + ipv4 + IPv4 subnet mask + + + + + + Gateway IP address + + + + + ipv4 + IPv4 address + + + + + Pool of client IPv4 addresses diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index ca47c32181d..fe04f7a20e6 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -627,5 +627,60 @@ def test_openvpn_site2site_interfaces_tun(self): self.assertNotIn(interface, interfaces()) + def test_openvpn_server_server_bridge(self): + # Create OpenVPN server interface using server-bridge. + # Validate configuration afterwards. + br_if = 'br0' + vtun_if = 'vtun5010' + auth_hash = 'sha256' + path = base_path + [vtun_if] + start_subnet = "192.168.0.100" + stop_subnet = "192.168.0.200" + mask_subnet = "255.255.255.0" + gw_subnet = "192.168.0.1" + + self.cli_set(['interfaces', 'bridge', br_if, 'member', 'interface', vtun_if]) + self.cli_set(path + ['device-type', 'tap']) + self.cli_set(path + ['encryption', 'data-ciphers', 'aes192']) + self.cli_set(path + ['hash', auth_hash]) + self.cli_set(path + ['mode', 'server']) + self.cli_set(path + ['server', 'server-bridge', 'gateway', gw_subnet]) + self.cli_set(path + ['server', 'server-bridge', 'start', start_subnet]) + self.cli_set(path + ['server', 'server-bridge', 'stop', stop_subnet]) + self.cli_set(path + ['server', 'server-bridge', 'subnet-mask', mask_subnet]) + self.cli_set(path + ['keep-alive', 'failure-count', '5']) + self.cli_set(path + ['keep-alive', 'interval', '5']) + self.cli_set(path + ['tls', 'ca-certificate', 'ovpn_test']) + self.cli_set(path + ['tls', 'certificate', 'ovpn_test']) + self.cli_set(path + ['tls', 'dh-params', 'ovpn_test']) + + self.cli_commit() + + + + config_file = f'/run/openvpn/{vtun_if}.conf' + config = read_file(config_file) + self.assertIn(f'dev {vtun_if}', config) + self.assertIn(f'dev-type tap', config) + self.assertIn(f'proto udp', config) # default protocol + self.assertIn(f'auth {auth_hash}', config) + self.assertIn(f'data-ciphers AES-192-CBC', config) + self.assertIn(f'mode server', config) + self.assertIn(f'server-bridge {gw_subnet} {mask_subnet} {start_subnet} {stop_subnet}', config) + elf.assertIn(f'keepalive 5 25', config) + + + + # TLS options + self.assertIn(f'ca /run/openvpn/{vtun_if}_ca.pem', config) + self.assertIn(f'cert /run/openvpn/{vtun_if}_cert.pem', config) + self.assertIn(f'key /run/openvpn/{vtun_if}_cert.key', config) + self.assertIn(f'dh /run/openvpn/{vtun_if}_dh.pem', config) + + # check that no interface remained after deleting them + self.cli_delete((['interfaces', 'bridge', br_if, 'member', 'interface', vtun_if]) + self.cli_delete(base_path) + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index a03bd595989..46bd6acf75c 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -378,6 +378,22 @@ def verify(openvpn): if (client_v.get('ip') and len(client_v['ip']) > 1) or (client_v.get('ipv6_ip') and len(client_v['ipv6_ip']) > 1): raise ConfigError(f'Server client "{client_k}": cannot specify more than 1 IPv4 and 1 IPv6 IP') + if dict_search('server.server_bridge', openvpn): + # check if server-bridge is a tap interfaces + if not openvpn['device_type'] == 'tap' and dict_search('server.server_bridge', openvpn): + raise ConfigError('Must specify "device-type tap" with server-bridge mode') + elif not (dict_search('server.server_bridge.start', openvpn) and dict_search('server.server_bridge.stop', openvpn)): + raise ConfigError('Server server-bridge requires both start and stop addresses') + else: + v4PoolStart = IPv4Address(dict_search('server.server_bridge.start', openvpn)) + v4PoolStop = IPv4Address(dict_search('server.server_bridge.stop', openvpn)) + if v4PoolStart > v4PoolStop: + raise ConfigError(f'Server server-bridge start address {v4PoolStart} is larger than stop address {v4PoolStop}') + + v4PoolSize = int(v4PoolStop) - int(v4PoolStart) + if v4PoolSize >= 65536: + raise ConfigError(f'Server server_bridge is too large [{v4PoolStart} -> {v4PoolStop} = {v4PoolSize}], maximum is 65536 addresses.') + if dict_search('server.client_ip_pool', openvpn): if not (dict_search('server.client_ip_pool.start', openvpn) and dict_search('server.client_ip_pool.stop', openvpn)): raise ConfigError('Server client-ip-pool requires both start and stop addresses') From 4055090a8d4fd64288eab7ae41aa9440f5de4ece Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 1 Aug 2024 08:46:51 +0200 Subject: [PATCH 23/64] console: T3334: remove unused directories imported from vyos.defaults --- src/conf_mode/system_console.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/conf_mode/system_console.py b/src/conf_mode/system_console.py index 27bf92e0b14..b380e0521c4 100755 --- a/src/conf_mode/system_console.py +++ b/src/conf_mode/system_console.py @@ -22,7 +22,6 @@ from vyos.utils.serial import restart_login_consoles from vyos.system import grub_util from vyos.template import render -from vyos.defaults import directories from vyos import ConfigError from vyos import airbag airbag.enable() From 20551379e8e2b4b6e342b39ea67738876e559bbf Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 24 Jul 2024 14:08:19 +0000 Subject: [PATCH 24/64] T4072: firewall: extend firewall bridge capabilities, in order to include new chains, priorities, and firewall groups --- data/templates/firewall/nftables-bridge.j2 | 85 +++++++++++++++ data/templates/firewall/nftables-defines.j2 | 10 +- data/templates/firewall/nftables.j2 | 101 +++++++++++++++++- interface-definitions/firewall.xml.in | 3 + .../include/firewall/address-inet.xml.i | 63 +++++++++++ .../include/firewall/address-mask-inet.xml.i | 19 ++++ .../include/firewall/bridge-custom-name.xml.i | 5 + .../firewall/bridge-hook-forward.xml.i | 5 + .../include/firewall/bridge-hook-input.xml.i | 39 +++++++ .../include/firewall/bridge-hook-output.xml.i | 39 +++++++ .../firewall/bridge-hook-prerouting.xml.i | 37 +++++++ .../include/firewall/common-rule-bridge.xml.i | 33 ++++-- .../firewall/set-packet-modifications.xml.i | 78 ++++++++++++++ .../source-destination-group-inet.xml.i | 50 +++++++++ .../include/policy/route-common.xml.i | 95 +--------------- python/vyos/firewall.py | 56 ++++++---- 16 files changed, 589 insertions(+), 129 deletions(-) create mode 100644 interface-definitions/include/firewall/address-inet.xml.i create mode 100644 interface-definitions/include/firewall/address-mask-inet.xml.i create mode 100644 interface-definitions/include/firewall/bridge-hook-input.xml.i create mode 100644 interface-definitions/include/firewall/bridge-hook-output.xml.i create mode 100644 interface-definitions/include/firewall/bridge-hook-prerouting.xml.i create mode 100644 interface-definitions/include/firewall/set-packet-modifications.xml.i create mode 100644 interface-definitions/include/firewall/source-destination-group-inet.xml.i diff --git a/data/templates/firewall/nftables-bridge.j2 b/data/templates/firewall/nftables-bridge.j2 index dec027bf98f..1975fb9b092 100644 --- a/data/templates/firewall/nftables-bridge.j2 +++ b/data/templates/firewall/nftables-bridge.j2 @@ -1,9 +1,13 @@ +{% import 'firewall/nftables-defines.j2' as group_tmpl %} {% macro bridge(bridge) %} {% set ns = namespace(sets=[]) %} {% if bridge.forward is vyos_defined %} {% for prior, conf in bridge.forward.items() %} chain VYOS_FORWARD_{{ prior }} { type filter hook forward priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} {% if conf.rule is vyos_defined %} {% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} {{ rule_conf | nft_rule('FWD', prior, rule_id, 'bri') }} @@ -17,6 +21,46 @@ {% endfor %} {% endif %} +{% if bridge.input is vyos_defined %} +{% for prior, conf in bridge.input.items() %} + chain VYOS_INPUT_{{ prior }} { + type filter hook input priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('INP', prior, rule_id, 'bri') }} +{% if rule_conf.recent is vyos_defined %} +{% set ns.sets = ns.sets + ['INP_' + prior + '_' + rule_id] %} +{% endif %} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('INP-filter', 'bri') }} + } +{% endfor %} +{% endif %} + +{% if bridge.output is vyos_defined %} +{% for prior, conf in bridge.output.items() %} + chain VYOS_OUTUT_{{ prior }} { + type filter hook output priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('OUT', prior, rule_id, 'bri') }} +{% if rule_conf.recent is vyos_defined %} +{% set ns.sets = ns.sets + ['OUT_' + prior + '_' + rule_id] %} +{% endif %} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('OUT-filter', 'bri') }} + } +{% endfor %} +{% endif %} + {% if bridge.name is vyos_defined %} {% for name_text, conf in bridge.name.items() %} chain NAME_{{ name_text }} { @@ -32,4 +76,45 @@ } {% endfor %} {% endif %} + +{% for set_name in ns.sets %} + set RECENT_{{ set_name }} { + type ipv4_addr + size 65535 + flags dynamic + } +{% endfor %} +{% for set_name in ip_fqdn %} + set FQDN_{{ set_name }} { + type ipv4_addr + flags interval + } +{% endfor %} +{% if geoip_updated.name is vyos_defined %} +{% for setname in geoip_updated.name %} + set {{ setname }} { + type ipv4_addr + flags interval + } +{% endfor %} +{% endif %} + +{{ group_tmpl.groups(group, False, True) }} +{{ group_tmpl.groups(group, True, True) }} + +{% if global_options.state_policy is vyos_defined %} + chain VYOS_STATE_POLICY { +{% if global_options.state_policy.established is vyos_defined %} + {{ global_options.state_policy.established | nft_state_policy('established') }} +{% endif %} +{% if global_options.state_policy.invalid is vyos_defined %} + {{ global_options.state_policy.invalid | nft_state_policy('invalid') }} +{% endif %} +{% if global_options.state_policy.related is vyos_defined %} + {{ global_options.state_policy.related | nft_state_policy('related') }} +{% endif %} + return + } +{% endif %} + {% endmacro %} diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2 index 8a75ab2d67b..fa6cd74c0c4 100644 --- a/data/templates/firewall/nftables-defines.j2 +++ b/data/templates/firewall/nftables-defines.j2 @@ -1,7 +1,7 @@ {% macro groups(group, is_ipv6, is_l3) %} {% if group is vyos_defined %} {% set ip_type = 'ipv6_addr' if is_ipv6 else 'ipv4_addr' %} -{% if group.address_group is vyos_defined and not is_ipv6 and is_l3 %} +{% if group.address_group is vyos_defined and not is_ipv6 %} {% for group_name, group_conf in group.address_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} set A_{{ group_name }} { @@ -14,7 +14,7 @@ } {% endfor %} {% endif %} -{% if group.ipv6_address_group is vyos_defined and is_ipv6 and is_l3 %} +{% if group.ipv6_address_group is vyos_defined and is_ipv6 %} {% for group_name, group_conf in group.ipv6_address_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} set A6_{{ group_name }} { @@ -46,7 +46,7 @@ } {% endfor %} {% endif %} -{% if group.network_group is vyos_defined and not is_ipv6 and is_l3 %} +{% if group.network_group is vyos_defined and not is_ipv6 %} {% for group_name, group_conf in group.network_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} set N_{{ group_name }} { @@ -59,7 +59,7 @@ } {% endfor %} {% endif %} -{% if group.ipv6_network_group is vyos_defined and is_ipv6 and is_l3 %} +{% if group.ipv6_network_group is vyos_defined and is_ipv6 %} {% for group_name, group_conf in group.ipv6_network_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} set N6_{{ group_name }} { @@ -72,7 +72,7 @@ } {% endfor %} {% endif %} -{% if group.port_group is vyos_defined and is_l3 %} +{% if group.port_group is vyos_defined %} {% for group_name, group_conf in group.port_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} set P_{{ group_name }} { diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 index 68a3bfd8781..82dcefac00a 100644 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -339,7 +339,104 @@ table ip6 vyos_filter { delete table bridge vyos_filter {% endif %} table bridge vyos_filter { -{{ bridge_tmpl.bridge(bridge) }} +{% if bridge is vyos_defined %} +{% if bridge.forward is vyos_defined %} +{% for prior, conf in bridge.forward.items() %} + chain VYOS_FORWARD_{{ prior }} { + type filter hook forward priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('FWD', prior, rule_id, 'bri') }} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('FWD-' + prior, 'bri') }} + } +{% endfor %} +{% endif %} + +{% if bridge.input is vyos_defined %} +{% for prior, conf in bridge.input.items() %} + chain VYOS_INPUT_{{ prior }} { + type filter hook input priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('INP', prior, rule_id, 'bri') }} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('INP-' + prior, 'bri') }} + } +{% endfor %} +{% endif %} + +{% if bridge.output is vyos_defined %} +{% for prior, conf in bridge.output.items() %} + chain VYOS_OUTUT_{{ prior }} { + type filter hook output priority {{ prior }}; policy accept; +{% if global_options.state_policy is vyos_defined %} + jump VYOS_STATE_POLICY +{% endif %} +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('OUT', prior, rule_id, 'bri') }} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('OUT-' + prior, 'bri') }} + } +{% endfor %} +{% endif %} + +{% if bridge.prerouting is vyos_defined %} +{% for prior, conf in bridge.prerouting.items() %} + chain VYOS_PREROUTING_{{ prior }} { + type filter hook prerouting priority {{ prior }}; policy accept; +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('PRE', prior, rule_id, 'bri') }} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('PRE-' + prior, 'bri') }} + } +{% endfor %} +{% endif %} + +{% if bridge.name is vyos_defined %} +{% for name_text, conf in bridge.name.items() %} + chain NAME_{{ name_text }} { +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('NAM', name_text, rule_id, 'bri') }} +{% if rule_conf.recent is vyos_defined %} +{% set ns.sets = ns.sets + ['NAM_' + name_text + '_' + rule_id] %} +{% endif %} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule(name_text, 'bri') }} + } +{% endfor %} +{% endif %} + +{% endif %} {{ group_tmpl.groups(group, False, False) }} +{{ group_tmpl.groups(group, True, False) }} -} +{% if global_options.state_policy is vyos_defined %} + chain VYOS_STATE_POLICY { +{% if global_options.state_policy.established is vyos_defined %} + {{ global_options.state_policy.established | nft_state_policy('established') }} +{% endif %} +{% if global_options.state_policy.invalid is vyos_defined %} + {{ global_options.state_policy.invalid | nft_state_policy('invalid') }} +{% endif %} +{% if global_options.state_policy.related is vyos_defined %} + {{ global_options.state_policy.related | nft_state_policy('related') }} +{% endif %} + return + } +{% endif %} +} \ No newline at end of file diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index dc4625af0be..816dd1855fc 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -367,6 +367,9 @@ #include + #include + #include + #include #include diff --git a/interface-definitions/include/firewall/address-inet.xml.i b/interface-definitions/include/firewall/address-inet.xml.i new file mode 100644 index 00000000000..02ed8f6e4e8 --- /dev/null +++ b/interface-definitions/include/firewall/address-inet.xml.i @@ -0,0 +1,63 @@ + + + + IP address, subnet, or range + + ipv4 + IPv4 address to match + + + ipv4net + IPv4 prefix to match + + + ipv4range + IPv4 address range to match + + + !ipv4 + Match everything except the specified address + + + !ipv4net + Match everything except the specified prefix + + + !ipv4range + Match everything except the specified range + + + ipv6net + Subnet to match + + + ipv6range + IP range to match + + + !ipv6 + Match everything except the specified address + + + !ipv6net + Match everything except the specified prefix + + + !ipv6range + Match everything except the specified range + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/address-mask-inet.xml.i b/interface-definitions/include/firewall/address-mask-inet.xml.i new file mode 100644 index 00000000000..e2a5927ab1c --- /dev/null +++ b/interface-definitions/include/firewall/address-mask-inet.xml.i @@ -0,0 +1,19 @@ + + + + IP mask + + ipv4 + IPv4 mask to apply + + + ipv6 + IP mask to apply + + + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/bridge-custom-name.xml.i b/interface-definitions/include/firewall/bridge-custom-name.xml.i index 654493c0e87..48d48949e12 100644 --- a/interface-definitions/include/firewall/bridge-custom-name.xml.i +++ b/interface-definitions/include/firewall/bridge-custom-name.xml.i @@ -32,6 +32,11 @@ #include + #include + #include + #include + #include + #include diff --git a/interface-definitions/include/firewall/bridge-hook-forward.xml.i b/interface-definitions/include/firewall/bridge-hook-forward.xml.i index 99f66ec772b..0bc1fc357b8 100644 --- a/interface-definitions/include/firewall/bridge-hook-forward.xml.i +++ b/interface-definitions/include/firewall/bridge-hook-forward.xml.i @@ -26,6 +26,11 @@ #include + #include + #include + #include + #include + #include diff --git a/interface-definitions/include/firewall/bridge-hook-input.xml.i b/interface-definitions/include/firewall/bridge-hook-input.xml.i new file mode 100644 index 00000000000..32de14d5417 --- /dev/null +++ b/interface-definitions/include/firewall/bridge-hook-input.xml.i @@ -0,0 +1,39 @@ + + + + Bridge input firewall + + + + + Bridge firewall input filter + + + #include + #include + #include + + + Bridge Firewall input filter rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + #include + #include + #include + + + + + + + diff --git a/interface-definitions/include/firewall/bridge-hook-output.xml.i b/interface-definitions/include/firewall/bridge-hook-output.xml.i new file mode 100644 index 00000000000..da0c02470c0 --- /dev/null +++ b/interface-definitions/include/firewall/bridge-hook-output.xml.i @@ -0,0 +1,39 @@ + + + + Bridge output firewall + + + + + Bridge firewall output filter + + + #include + #include + #include + + + Bridge Firewall output filter rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + #include + #include + #include + + + + + + + diff --git a/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i b/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i new file mode 100644 index 00000000000..b6c1fe87a7b --- /dev/null +++ b/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i @@ -0,0 +1,37 @@ + + + + Bridge prerouting firewall + + + + + Bridge firewall prerouting filter + + + #include + #include + #include + + + Bridge Firewall prerouting filter rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + #include + + + + + + + diff --git a/interface-definitions/include/firewall/common-rule-bridge.xml.i b/interface-definitions/include/firewall/common-rule-bridge.xml.i index dcdd970acdf..b47408aa83d 100644 --- a/interface-definitions/include/firewall/common-rule-bridge.xml.i +++ b/interface-definitions/include/firewall/common-rule-bridge.xml.i @@ -1,15 +1,37 @@ +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include Destination parameters #include + #include + #include + #include + #include -#include Set jump target. Action jump must be defined to use this setting @@ -18,17 +40,16 @@ -#include -#include Source parameters #include + #include + #include + #include + #include -#include -#include -#include diff --git a/interface-definitions/include/firewall/set-packet-modifications.xml.i b/interface-definitions/include/firewall/set-packet-modifications.xml.i new file mode 100644 index 00000000000..eda568a0ebe --- /dev/null +++ b/interface-definitions/include/firewall/set-packet-modifications.xml.i @@ -0,0 +1,78 @@ + + + + Packet modifications + + + + + Connection marking + + u32:0-2147483647 + Connection marking + + + + + + + + + Packet Differentiated Services Codepoint (DSCP) + + u32:0-63 + DSCP number + + + + + + + + + Packet marking + + u32:1-2147483647 + Packet marking + + + + + + + + + Routing table to forward packet with + + u32:1-200 + Table number + + + main + Main table + + + + (main) + + + main + protocols static table + + + + + + TCP Maximum Segment Size + + u32:500-1460 + Explicitly set TCP MSS value + + + + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/source-destination-group-inet.xml.i b/interface-definitions/include/firewall/source-destination-group-inet.xml.i new file mode 100644 index 00000000000..174051624d1 --- /dev/null +++ b/interface-definitions/include/firewall/source-destination-group-inet.xml.i @@ -0,0 +1,50 @@ + + + + Group + + + + + Group of IPv4 addresses + + firewall group address-group + + + + + + Group of IPv6 addresses + + firewall group ipv6-address-group + + + + #include + + + Group of IPv4 networks + + firewall group network-group + + + + + + Group of IPv6 networks + + firewall group ipv6-network-group + + + + + + Group of ports + + firewall group port-group + + + + + + diff --git a/interface-definitions/include/policy/route-common.xml.i b/interface-definitions/include/policy/route-common.xml.i index 203be73e759..19ffc05069a 100644 --- a/interface-definitions/include/policy/route-common.xml.i +++ b/interface-definitions/include/policy/route-common.xml.i @@ -66,100 +66,7 @@ - - - Packet modifications - - - - - Connection marking - - u32:0-2147483647 - Connection marking - - - - - - - - - Packet Differentiated Services Codepoint (DSCP) - - u32:0-63 - DSCP number - - - - - - - - - Packet marking - - u32:1-2147483647 - Packet marking - - - - - - - - - Routing table to forward packet with - - u32:1-200 - Table number - - - main - Main table - - - - (main) - - - main - protocols static table - - - - - - VRF to forward packet with - - txt - VRF instance name - - - default - Forward into default global VRF - - - default - vrf name - - #include - - - - - TCP Maximum Segment Size - - u32:500-1460 - Explicitly set TCP MSS value - - - - - - - - +#include #include #include #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index facd498ca10..cac6d2953cb 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -167,7 +167,10 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if address_mask: operator = '!=' if exclude else '==' operator = f'& {address_mask} {operator} ' - output.append(f'{ip_name} {prefix}addr {operator}{suffix}') + if is_ipv4(suffix): + output.append(f'ip {prefix}addr {operator}{suffix}') + else: + output.append(f'ip6 {prefix}addr {operator}{suffix}') if 'fqdn' in side_conf: fqdn = side_conf['fqdn'] @@ -236,22 +239,38 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if 'group' in side_conf: group = side_conf['group'] - if 'address_group' in group: - group_name = group['address_group'] - operator = '' - exclude = group_name[0] == "!" - if exclude: - operator = '!=' - group_name = group_name[1:] - if address_mask: - operator = '!=' if exclude else '==' - operator = f'& {address_mask} {operator}' - output.append(f'{ip_name} {prefix}addr {operator} @A{def_suffix}_{group_name}') - elif 'dynamic_address_group' in group: + for ipvx_address_group in ['address_group', 'ipv4_address_group', 'ipv6_address_group']: + if ipvx_address_group in group: + group_name = group[ipvx_address_group] + operator = '' + exclude = group_name[0] == "!" + if exclude: + operator = '!=' + group_name = group_name[1:] + if address_mask: + operator = '!=' if exclude else '==' + operator = f'& {address_mask} {operator}' + # for bridge, change ip_name + if ip_name == 'bri': + ip_name = 'ip' if ipvx_address_group == 'ipv4_address_group' else 'ip6' + def_suffix = '6' if ipvx_address_group == 'ipv6_address_group' else '' + output.append(f'{ip_name} {prefix}addr {operator} @A{def_suffix}_{group_name}') + for ipvx_network_group in ['network_group', 'ipv4_network_group', 'ipv6_network_group']: + if ipvx_network_group in group: + group_name = group[ipvx_network_group] + operator = '' + if group_name[0] == "!": + operator = '!=' + group_name = group_name[1:] + # for bridge, change ip_name + if ip_name == 'bri': + ip_name = 'ip' if ipvx_network_group == 'ipv4_network_group' else 'ip6' + def_suffix = '6' if ipvx_network_group == 'ipv6_network_group' else '' + output.append(f'{ip_name} {prefix}addr {operator} @N{def_suffix}_{group_name}') + if 'dynamic_address_group' in group: group_name = group['dynamic_address_group'] operator = '' - exclude = group_name[0] == "!" - if exclude: + if group_name[0] == "!": operator = '!=' group_name = group_name[1:] output.append(f'{ip_name} {prefix}addr {operator} @DA{def_suffix}_{group_name}') @@ -263,13 +282,6 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): operator = '!=' group_name = group_name[1:] output.append(f'{ip_name} {prefix}addr {operator} @D_{group_name}') - elif 'network_group' in group: - group_name = group['network_group'] - operator = '' - if group_name[0] == '!': - operator = '!=' - group_name = group_name[1:] - output.append(f'{ip_name} {prefix}addr {operator} @N{def_suffix}_{group_name}') if 'mac_group' in group: group_name = group['mac_group'] operator = '' From 7a18c719df1b3f2515baff8bdecc8784f1d935b1 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 24 Jul 2024 14:15:34 +0000 Subject: [PATCH 25/64] T4072: firewall: improve error handling when firewall configuration is wrong. Use nft -c option to check temporary file, and use output provided by nftables to parse the error if possible, or print it as it is if it's an unknown error --- src/conf_mode/firewall.py | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 352d5cbb174..77218cc7738 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -36,6 +36,7 @@ from vyos.utils.process import rc_cmd from vyos import ConfigError from vyos import airbag +from subprocess import run as subp_run airbag.enable() @@ -495,8 +496,53 @@ def generate(firewall): render(sysctl_file, 'firewall/sysctl-firewall.conf.j2', firewall) return None +def parse_firewall_error(output): + # Define the regex patterns to extract the error message and the comment + error_pattern = re.compile(r'Error:\s*(.*?)\n') + comment_pattern = re.compile(r'comment\s+"([^"]+)"') + error_output = [] + + # Find all error messages in the output + error_matches = error_pattern.findall(output) + # Find all comment matches in the output + comment_matches = comment_pattern.findall(output) + + if not error_matches or not comment_matches: + raise ConfigError(f'Unknown firewall error detected: {output}') + + error_output.append('Fail to apply firewall') + # Loop over the matches and process them + for error_message, comment in zip(error_matches, comment_matches): + # Parse the comment + parsed_entries = comment.split('-') + family = 'bridge' if parsed_entries[0] == 'bri' else parsed_entries[0] + if parsed_entries[1] == 'NAM': + chain = 'name' + elif parsed_entries[1] == 'FWD': + chain = 'forward' + elif parsed_entries[1] == 'INP': + chain = 'input' + elif parsed_entries[1] == 'OUT': + chain = 'output' + elif parsed_entries[1] == 'PRE': + chain = 'prerouting' + error_output.append(f'Error found on: firewall {family} {chain} {parsed_entries[2]} rule {parsed_entries[3]}') + error_output.append(f'\tError message: {error_message.strip()}') + + raise ConfigError('\n'.join(error_output)) + def apply(firewall): + # Use nft -c option to check current configuration file + completed_process = subp_run(['nft', '-c', '--file', nftables_conf], capture_output=True) + install_result = completed_process.returncode + if install_result == 1: + # We need to handle firewall error + output = completed_process.stderr + parse_firewall_error(output.decode()) + + # No error detected during check, we can apply the new configuration install_result, output = rc_cmd(f'nft --file {nftables_conf}') + # Double check just in case if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') From a8a9cfe750da719605ab90ce8c83c42276ab07f3 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 24 Jul 2024 17:40:28 +0000 Subject: [PATCH 26/64] T6570: firewall: add global-option to configure sysctl parameter for enabling/disabling sending traffic from bridge layer to ipvX layer --- .../firewall/sysctl-firewall.conf.j2 | 8 ++++++++ .../include/firewall/global-options.xml.i | 19 +++++++++++++++++++ src/etc/sysctl.d/30-vyos-router.conf | 5 +++++ 3 files changed, 32 insertions(+) diff --git a/data/templates/firewall/sysctl-firewall.conf.j2 b/data/templates/firewall/sysctl-firewall.conf.j2 index b9c3311e27b..119c6577bbd 100644 --- a/data/templates/firewall/sysctl-firewall.conf.j2 +++ b/data/templates/firewall/sysctl-firewall.conf.j2 @@ -13,6 +13,14 @@ net.ipv4.conf.*.send_redirects = {{ 1 if global_options.send_redirects == 'enabl net.ipv4.tcp_syncookies = {{ 1 if global_options.syn_cookies == 'enable' else 0 }} net.ipv4.tcp_rfc1337 = {{ 1 if global_options.twa_hazards_protection == 'enable' else 0 }} +{% if global_options.apply_for_bridge is vyos_defined %} +net.bridge.bridge-nf-call-iptables = {{ 1 if global_options.apply_for_bridge.ipv4 is vyos_defined else 0 }} +net.bridge.bridge-nf-call-ip6tables = {{ 1 if global_options.apply_for_bridge.ipv6 is vyos_defined else 0 }} +{% else %} +net.bridge.bridge-nf-call-iptables =0 +net.bridge.bridge-nf-call-ip6tables = 0 +{% endif %} + ## Timeout values: net.netfilter.nf_conntrack_icmp_timeout = {{ global_options.timeout.icmp }} net.netfilter.nf_conntrack_generic_timeout = {{ global_options.timeout.other }} diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i index 9039b76fd63..1f289967256 100644 --- a/interface-definitions/include/firewall/global-options.xml.i +++ b/interface-definitions/include/firewall/global-options.xml.i @@ -44,6 +44,25 @@ disable + + + Apply configured firewall rules to traffic switched by bridges + + + + + Apply configured IPv4 firewall rules + + + + + + Apply configured IPv6 firewall rules + + + + + Policy for handling IPv4 directed broadcast forwarding on all interfaces diff --git a/src/etc/sysctl.d/30-vyos-router.conf b/src/etc/sysctl.d/30-vyos-router.conf index c9b8ef8fe43..76be41ddc7e 100644 --- a/src/etc/sysctl.d/30-vyos-router.conf +++ b/src/etc/sysctl.d/30-vyos-router.conf @@ -110,3 +110,8 @@ net.ipv6.conf.all.seg6_enabled = 0 net.ipv6.conf.default.seg6_enabled = 0 net.vrf.strict_mode = 1 + +# https://vyos.dev/T6570 +# By default, do not forward traffic from bridge to IPvX layer +net.bridge.bridge-nf-call-iptables = 0 +net.bridge.bridge-nf-call-ip6tables = 0 \ No newline at end of file From fa764927c14350104671edbb2bb3570ab267e416 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Mon, 29 Jul 2024 17:55:56 +0000 Subject: [PATCH 27/64] T4072: firewall: extend firewall bridge smoketest --- .../firewall/sysctl-firewall.conf.j2 | 2 +- smoketest/scripts/cli/test_firewall.py | 34 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/data/templates/firewall/sysctl-firewall.conf.j2 b/data/templates/firewall/sysctl-firewall.conf.j2 index 119c6577bbd..ae6a8969c00 100644 --- a/data/templates/firewall/sysctl-firewall.conf.j2 +++ b/data/templates/firewall/sysctl-firewall.conf.j2 @@ -17,7 +17,7 @@ net.ipv4.tcp_rfc1337 = {{ 1 if global_options.twa_hazards_protection == 'enable' net.bridge.bridge-nf-call-iptables = {{ 1 if global_options.apply_for_bridge.ipv4 is vyos_defined else 0 }} net.bridge.bridge-nf-call-ip6tables = {{ 1 if global_options.apply_for_bridge.ipv6 is vyos_defined else 0 }} {% else %} -net.bridge.bridge-nf-call-iptables =0 +net.bridge.bridge-nf-call-iptables = 0 net.bridge.bridge-nf-call-ip6tables = 0 {% endif %} diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index e6317050c14..d2826a8bd03 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -695,13 +695,21 @@ def test_ipv4_state_and_status_rules(self): self.verify_nftables_chain([['accept']], 'ip vyos_conntrack', 'FW_CONNTRACK') self.verify_nftables_chain([['return']], 'ip6 vyos_conntrack', 'FW_CONNTRACK') - def test_bridge_basic_rules(self): + def test_bridge_firewall(self): name = 'smoketest' interface_in = 'eth0' mac_address = '00:53:00:00:00:01' vlan_id = '12' vlan_prior = '3' + # Check bridge-nf-call-iptables default value: 0 + self.assertEqual(get_sysctl('net.bridge.bridge-nf-call-iptables'), '0') + self.assertEqual(get_sysctl('net.bridge.bridge-nf-call-ip6tables'), '0') + + self.cli_set(['firewall', 'group', 'ipv6-address-group', 'AGV6', 'address', '2001:db1::1']) + self.cli_set(['firewall', 'global-options', 'state-policy', 'established', 'action', 'accept']) + self.cli_set(['firewall', 'global-options', 'apply-for-bridge', 'ipv4']) + self.cli_set(['firewall', 'bridge', 'name', name, 'default-action', 'accept']) self.cli_set(['firewall', 'bridge', 'name', name, 'default-log']) self.cli_set(['firewall', 'bridge', 'name', name, 'rule', '1', 'action', 'accept']) @@ -718,20 +726,42 @@ def test_bridge_basic_rules(self): self.cli_set(['firewall', 'bridge', 'forward', 'filter', 'rule', '2', 'jump-target', name]) self.cli_set(['firewall', 'bridge', 'forward', 'filter', 'rule', '2', 'vlan', 'priority', vlan_prior]) + self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'action', 'accept']) + self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'inbound-interface', 'name', interface_in]) + self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'source', 'address', '192.0.2.2']) + self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'state', 'new']) + + self.cli_set(['firewall', 'bridge', 'prerouting', 'filter', 'rule', '1', 'action', 'drop']) + self.cli_set(['firewall', 'bridge', 'prerouting', 'filter', 'rule', '1', 'destination', 'group', 'ipv6-address-group', 'AGV6']) + + self.cli_commit() nftables_search = [ + ['set A6_AGV6'], + ['type ipv6_addr'], + ['elements', '2001:db1::1'], ['chain VYOS_FORWARD_filter'], ['type filter hook forward priority filter; policy accept;'], + ['jump VYOS_STATE_POLICY'], [f'vlan id {vlan_id}', 'accept'], [f'vlan pcp {vlan_prior}', f'jump NAME_{name}'], ['log prefix "[bri-FWD-filter-default-D]"', 'drop', 'FWD-filter default-action drop'], [f'chain NAME_{name}'], [f'ether saddr {mac_address}', f'iifname "{interface_in}"', f'log prefix "[bri-NAM-{name}-1-A]" log level crit', 'accept'], - ['accept', f'{name} default-action accept'] + ['accept', f'{name} default-action accept'], + ['chain VYOS_INPUT_filter'], + ['type filter hook input priority filter; policy accept;'], + ['ct state new', 'ip saddr 192.0.2.2', f'iifname "{interface_in}"', 'accept'], + ['chain VYOS_PREROUTING_filter'], + ['type filter hook prerouting priority filter; policy accept;'], + ['ip6 daddr @A6_AGV6', 'drop'] ] self.verify_nftables(nftables_search, 'bridge vyos_filter') + ## Check bridge-nf-call-iptables is set to 1, and for ipv6 remains on default 0 + self.assertEqual(get_sysctl('net.bridge.bridge-nf-call-iptables'), '1') + self.assertEqual(get_sysctl('net.bridge.bridge-nf-call-ip6tables'), '0') def test_source_validation(self): # Strict From c33cd6157ebc5c08dc1e3ff1aa36f2d2fbb9ca83 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 31 Jul 2024 12:42:25 +0000 Subject: [PATCH 28/64] T4072: change same helpers in xml definitions; add notrack action for prerouting chain; re introduce in policy; change global options for passing traffic to IPvX firewall; update smoketest --- .../firewall/sysctl-firewall.conf.j2 | 6 +- .../include/firewall/bridge-custom-name.xml.i | 1 + .../firewall/bridge-hook-forward.xml.i | 1 + .../include/firewall/bridge-hook-input.xml.i | 1 + .../include/firewall/bridge-hook-output.xml.i | 1 + .../firewall/bridge-hook-prerouting.xml.i | 4 +- .../include/firewall/common-rule-bridge.xml.i | 1 - .../include/firewall/global-options.xml.i | 2 +- .../firewall/set-packet-modifications.xml.i | 32 +++-- smoketest/scripts/cli/test_firewall.py | 7 +- src/conf_mode/firewall.py | 117 ++++++++---------- 11 files changed, 91 insertions(+), 82 deletions(-) diff --git a/data/templates/firewall/sysctl-firewall.conf.j2 b/data/templates/firewall/sysctl-firewall.conf.j2 index ae6a8969c00..6c33ffdc85d 100644 --- a/data/templates/firewall/sysctl-firewall.conf.j2 +++ b/data/templates/firewall/sysctl-firewall.conf.j2 @@ -13,9 +13,9 @@ net.ipv4.conf.*.send_redirects = {{ 1 if global_options.send_redirects == 'enabl net.ipv4.tcp_syncookies = {{ 1 if global_options.syn_cookies == 'enable' else 0 }} net.ipv4.tcp_rfc1337 = {{ 1 if global_options.twa_hazards_protection == 'enable' else 0 }} -{% if global_options.apply_for_bridge is vyos_defined %} -net.bridge.bridge-nf-call-iptables = {{ 1 if global_options.apply_for_bridge.ipv4 is vyos_defined else 0 }} -net.bridge.bridge-nf-call-ip6tables = {{ 1 if global_options.apply_for_bridge.ipv6 is vyos_defined else 0 }} +{% if global_options.apply_to_bridged_traffic is vyos_defined %} +net.bridge.bridge-nf-call-iptables = {{ 1 if global_options.apply_to_bridged_traffic.ipv4 is vyos_defined else 0 }} +net.bridge.bridge-nf-call-ip6tables = {{ 1 if global_options.apply_to_bridged_traffic.ipv6 is vyos_defined else 0 }} {% else %} net.bridge.bridge-nf-call-iptables = 0 net.bridge.bridge-nf-call-ip6tables = 0 diff --git a/interface-definitions/include/firewall/bridge-custom-name.xml.i b/interface-definitions/include/firewall/bridge-custom-name.xml.i index 48d48949e12..9a2a829d069 100644 --- a/interface-definitions/include/firewall/bridge-custom-name.xml.i +++ b/interface-definitions/include/firewall/bridge-custom-name.xml.i @@ -32,6 +32,7 @@ #include + #include #include #include #include diff --git a/interface-definitions/include/firewall/bridge-hook-forward.xml.i b/interface-definitions/include/firewall/bridge-hook-forward.xml.i index 0bc1fc357b8..fcc9819254f 100644 --- a/interface-definitions/include/firewall/bridge-hook-forward.xml.i +++ b/interface-definitions/include/firewall/bridge-hook-forward.xml.i @@ -26,6 +26,7 @@ #include + #include #include #include #include diff --git a/interface-definitions/include/firewall/bridge-hook-input.xml.i b/interface-definitions/include/firewall/bridge-hook-input.xml.i index 32de14d5417..f6a11f8dac2 100644 --- a/interface-definitions/include/firewall/bridge-hook-input.xml.i +++ b/interface-definitions/include/firewall/bridge-hook-input.xml.i @@ -26,6 +26,7 @@ #include + #include #include #include #include diff --git a/interface-definitions/include/firewall/bridge-hook-output.xml.i b/interface-definitions/include/firewall/bridge-hook-output.xml.i index da0c02470c0..38b8b08cad7 100644 --- a/interface-definitions/include/firewall/bridge-hook-output.xml.i +++ b/interface-definitions/include/firewall/bridge-hook-output.xml.i @@ -26,6 +26,7 @@ #include + #include #include #include #include diff --git a/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i b/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i index b6c1fe87a7b..ea567644f7a 100644 --- a/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i +++ b/interface-definitions/include/firewall/bridge-hook-prerouting.xml.i @@ -14,7 +14,7 @@ #include - Bridge Firewall prerouting filter rule number + Bridge firewall prerouting filter rule number u32:1-999999 Number for this firewall rule @@ -26,7 +26,7 @@ #include - #include + #include #include diff --git a/interface-definitions/include/firewall/common-rule-bridge.xml.i b/interface-definitions/include/firewall/common-rule-bridge.xml.i index b47408aa83d..9ae28f7bee1 100644 --- a/interface-definitions/include/firewall/common-rule-bridge.xml.i +++ b/interface-definitions/include/firewall/common-rule-bridge.xml.i @@ -1,7 +1,6 @@ #include #include -#include #include #include #include diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i index 1f289967256..cee8f1854d6 100644 --- a/interface-definitions/include/firewall/global-options.xml.i +++ b/interface-definitions/include/firewall/global-options.xml.i @@ -44,7 +44,7 @@ disable - + Apply configured firewall rules to traffic switched by bridges diff --git a/interface-definitions/include/firewall/set-packet-modifications.xml.i b/interface-definitions/include/firewall/set-packet-modifications.xml.i index eda568a0ebe..ee019b64ee7 100644 --- a/interface-definitions/include/firewall/set-packet-modifications.xml.i +++ b/interface-definitions/include/firewall/set-packet-modifications.xml.i @@ -6,10 +6,10 @@ - Connection marking + Set connection mark u32:0-2147483647 - Connection marking + Connection mark @@ -18,7 +18,7 @@ - Packet Differentiated Services Codepoint (DSCP) + Set DSCP (Packet Differentiated Services Codepoint) bits u32:0-63 DSCP number @@ -30,10 +30,10 @@ - Packet marking + Set packet mark u32:1-2147483647 - Packet marking + Packet mark @@ -42,7 +42,7 @@ - Routing table to forward packet with + Set the routing table for matched packets u32:1-200 Table number @@ -61,9 +61,27 @@ + + + VRF to forward packet with + + txt + VRF instance name + + + default + Forward into default global VRF + + + default + vrf name + + #include + + - TCP Maximum Segment Size + Set TCP Maximum Segment Size u32:500-1460 Explicitly set TCP MSS value diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index d2826a8bd03..551f8ce654f 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -708,7 +708,7 @@ def test_bridge_firewall(self): self.cli_set(['firewall', 'group', 'ipv6-address-group', 'AGV6', 'address', '2001:db1::1']) self.cli_set(['firewall', 'global-options', 'state-policy', 'established', 'action', 'accept']) - self.cli_set(['firewall', 'global-options', 'apply-for-bridge', 'ipv4']) + self.cli_set(['firewall', 'global-options', 'apply-to-bridged-traffic', 'ipv4']) self.cli_set(['firewall', 'bridge', 'name', name, 'default-action', 'accept']) self.cli_set(['firewall', 'bridge', 'name', name, 'default-log']) @@ -731,10 +731,9 @@ def test_bridge_firewall(self): self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'source', 'address', '192.0.2.2']) self.cli_set(['firewall', 'bridge', 'input', 'filter', 'rule', '1', 'state', 'new']) - self.cli_set(['firewall', 'bridge', 'prerouting', 'filter', 'rule', '1', 'action', 'drop']) + self.cli_set(['firewall', 'bridge', 'prerouting', 'filter', 'rule', '1', 'action', 'notrack']) self.cli_set(['firewall', 'bridge', 'prerouting', 'filter', 'rule', '1', 'destination', 'group', 'ipv6-address-group', 'AGV6']) - self.cli_commit() nftables_search = [ @@ -755,7 +754,7 @@ def test_bridge_firewall(self): ['ct state new', 'ip saddr 192.0.2.2', f'iifname "{interface_in}"', 'accept'], ['chain VYOS_PREROUTING_filter'], ['type filter hook prerouting priority filter; policy accept;'], - ['ip6 daddr @A6_AGV6', 'drop'] + ['ip6 daddr @A6_AGV6', 'notrack'] ] self.verify_nftables(nftables_search, 'bridge vyos_filter') diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 77218cc7738..02bf00bcc1f 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -48,7 +48,12 @@ 'domain_group', 'network_group', 'port_group', - 'interface_group' + 'interface_group', + ## Added for group ussage in bridge firewall + 'ipv4_address_group', + 'ipv6_address_group', + 'ipv4_network_group', + 'ipv6_network_group' ] nested_group_types = [ @@ -129,41 +134,32 @@ def get_config(config=None): return firewall -def verify_jump_target(firewall, root_chain, jump_target, ipv6, recursive=False): +def verify_jump_target(firewall, hook, jump_target, family, recursive=False): targets_seen = [] targets_pending = [jump_target] while targets_pending: target = targets_pending.pop() - if not ipv6: - if target not in dict_search_args(firewall, 'ipv4', 'name'): - raise ConfigError(f'Invalid jump-target. Firewall name {target} does not exist on the system') - target_rules = dict_search_args(firewall, 'ipv4', 'name', target, 'rule') - else: - if target not in dict_search_args(firewall, 'ipv6', 'name'): - raise ConfigError(f'Invalid jump-target. Firewall ipv6 name {target} does not exist on the system') - target_rules = dict_search_args(firewall, 'ipv6', 'name', target, 'rule') + if 'name' not in firewall[family]: + raise ConfigError(f'Invalid jump-target. Firewall {family} name {target} does not exist on the system') + elif target not in dict_search_args(firewall, family, 'name'): + raise ConfigError(f'Invalid jump-target. Firewall {family} name {target} does not exist on the system') - no_ipsec_in = root_chain in ('output', ) + target_rules = dict_search_args(firewall, family, 'name', target, 'rule') + no_ipsec_in = hook in ('output', ) if target_rules: for target_rule_conf in target_rules.values(): # Output hook types will not tolerate 'meta ipsec exists' matches even in jump targets: if no_ipsec_in and (dict_search_args(target_rule_conf, 'ipsec', 'match_ipsec_in') is not None \ or dict_search_args(target_rule_conf, 'ipsec', 'match_none_in') is not None): - if not ipv6: - raise ConfigError(f'Invalid jump-target for {root_chain}. Firewall name {target} rules contain incompatible ipsec inbound matches') - else: - raise ConfigError(f'Invalid jump-target for {root_chain}. Firewall ipv6 name {target} rules contain incompatible ipsec inbound matches') + raise ConfigError(f'Invalid jump-target for {hook}. Firewall {family} name {target} rules contain incompatible ipsec inbound matches') # Make sure we're not looping back on ourselves somewhere: if recursive and 'jump_target' in target_rule_conf: child_target = target_rule_conf['jump_target'] if child_target in targets_seen: - if not ipv6: - raise ConfigError(f'Loop detected in jump-targets, firewall name {target} refers to previously traversed name {child_target}') - else: - raise ConfigError(f'Loop detected in jump-targets, firewall ipv6 name {target} refers to previously traversed ipv6 name {child_target}') + raise ConfigError(f'Loop detected in jump-targets, firewall {family} name {target} refers to previously traversed {family} name {child_target}') targets_pending.append(child_target) if len(targets_seen) == 7: path_txt = ' -> '.join(targets_seen) @@ -171,7 +167,7 @@ def verify_jump_target(firewall, root_chain, jump_target, ipv6, recursive=False) targets_seen.append(target) -def verify_rule(firewall, chain_name, rule_conf, ipv6): +def verify_rule(firewall, family, hook, priority, rule_id, rule_conf): if 'action' not in rule_conf: raise ConfigError('Rule action must be defined') @@ -182,10 +178,10 @@ def verify_rule(firewall, chain_name, rule_conf, ipv6): if 'jump' not in rule_conf['action']: raise ConfigError('jump-target defined, but action jump needed and it is not defined') target = rule_conf['jump_target'] - if chain_name != 'name': # This is a bit clumsy, but consolidates a chunk of code. - verify_jump_target(firewall, chain_name, target, ipv6, recursive=True) + if hook != 'name': # This is a bit clumsy, but consolidates a chunk of code. + verify_jump_target(firewall, hook, target, family, recursive=True) else: - verify_jump_target(firewall, chain_name, target, ipv6, recursive=False) + verify_jump_target(firewall, hook, target, family, recursive=False) if rule_conf['action'] == 'offload': if 'offload_target' not in rule_conf: @@ -247,9 +243,9 @@ def verify_rule(firewall, chain_name, rule_conf, ipv6): raise ConfigError(f'Cannot match a tcp flag as set and not set') if 'protocol' in rule_conf: - if rule_conf['protocol'] == 'icmp' and ipv6: + if rule_conf['protocol'] == 'icmp' and family == 'ipv6': raise ConfigError(f'Cannot match IPv4 ICMP protocol on IPv6, use ipv6-icmp') - if rule_conf['protocol'] == 'ipv6-icmp' and not ipv6: + if rule_conf['protocol'] == 'ipv6-icmp' and family == 'ipv4': raise ConfigError(f'Cannot match IPv6 ICMP protocol on IPv4, use icmp') for side in ['destination', 'source']: @@ -267,7 +263,18 @@ def verify_rule(firewall, chain_name, rule_conf, ipv6): if group in side_conf['group']: group_name = side_conf['group'][group] - fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group + if family == 'ipv6' and group in ['address_group', 'network_group']: + fw_group = f'ipv6_{group}' + elif family == 'bridge': + if group =='ipv4_address_group': + fw_group = 'address_group' + elif group == 'ipv4_network_group': + fw_group = 'network_group' + else: + fw_group = group + else: + fw_group = group + error_group = fw_group.replace("_", "-") if group in ['address_group', 'network_group', 'domain_group']: @@ -303,7 +310,7 @@ def verify_rule(firewall, chain_name, rule_conf, ipv6): raise ConfigError(f'Dynamic address group must be defined.') else: target = rule_conf['add_address_to_group'][type]['address_group'] - fwall_group = 'ipv6_address_group' if ipv6 else 'address_group' + fwall_group = 'ipv6_address_group' if family == 'ipv6' else 'address_group' group_obj = dict_search_args(firewall, 'group', 'dynamic_group', fwall_group, target) if group_obj is None: raise ConfigError(f'Invalid dynamic address group on firewall rule') @@ -380,43 +387,25 @@ def verify(firewall): for group_name, group in groups.items(): verify_nested_group(group_name, group, groups, []) - if 'ipv4' in firewall: - for name in ['name','forward','input','output', 'prerouting']: - if name in firewall['ipv4']: - for name_id, name_conf in firewall['ipv4'][name].items(): - if 'jump' in name_conf['default_action'] and 'default_jump_target' not in name_conf: - raise ConfigError('default-action set to jump, but no default-jump-target specified') - if 'default_jump_target' in name_conf: - target = name_conf['default_jump_target'] - if 'jump' not in name_conf['default_action']: - raise ConfigError('default-jump-target defined, but default-action jump needed and it is not defined') - if name_conf['default_jump_target'] == name_id: - raise ConfigError(f'Loop detected on default-jump-target.') - verify_jump_target(firewall, name, target, False, recursive=True) - - if 'rule' in name_conf: - for rule_id, rule_conf in name_conf['rule'].items(): - verify_rule(firewall, name, rule_conf, False) - - if 'ipv6' in firewall: - for name in ['name','forward','input','output', 'prerouting']: - if name in firewall['ipv6']: - for name_id, name_conf in firewall['ipv6'][name].items(): - if 'jump' in name_conf['default_action'] and 'default_jump_target' not in name_conf: - raise ConfigError('default-action set to jump, but no default-jump-target specified') - if 'default_jump_target' in name_conf: - target = name_conf['default_jump_target'] - if 'jump' not in name_conf['default_action']: - raise ConfigError('default-jump-target defined, but default-action jump needed and it is not defined') - if name_conf['default_jump_target'] == name_id: - raise ConfigError(f'Loop detected on default-jump-target.') - verify_jump_target(firewall, name, target, True, recursive=True) - - if 'rule' in name_conf: - for rule_id, rule_conf in name_conf['rule'].items(): - verify_rule(firewall, name, rule_conf, True) - - #### ZONESSSS + for family in ['ipv4', 'ipv6', 'bridge']: + if family in firewall: + for chain in ['name','forward','input','output', 'prerouting']: + if chain in firewall[family]: + for priority, priority_conf in firewall[family][chain].items(): + if 'jump' in priority_conf['default_action'] and 'default_jump_target' not in priority_conf: + raise ConfigError('default-action set to jump, but no default-jump-target specified') + if 'default_jump_target' in priority_conf: + target = priority_conf['default_jump_target'] + if 'jump' not in priority_conf['default_action']: + raise ConfigError('default-jump-target defined, but default-action jump needed and it is not defined') + if priority_conf['default_jump_target'] == priority: + raise ConfigError(f'Loop detected on default-jump-target.') + if target not in dict_search_args(firewall[family], 'name'): + raise ConfigError(f'Invalid jump-target. Firewall name {target} does not exist on the system') + if 'rule' in priority_conf: + for rule_id, rule_conf in priority_conf['rule'].items(): + verify_rule(firewall, family, chain, priority, rule_id, rule_conf) + local_zone = False zone_interfaces = [] From aeb51976ea23d68d35685bdaa535042a05016185 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 1 Aug 2024 11:47:31 -0500 Subject: [PATCH 29/64] nat64: T6627: call check_kmod within standard config function Functions called from config scripts outside of the standard functions get_config/verify/generate/apply will not be called when run under configd. Move as appropriate for the general config script structure and the specific script requirements. --- src/conf_mode/nat64.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/conf_mode/nat64.py b/src/conf_mode/nat64.py index 32a1c47d197..df501ce7f74 100755 --- a/src/conf_mode/nat64.py +++ b/src/conf_mode/nat64.py @@ -46,7 +46,12 @@ def get_config(config: Config | None = None) -> None: base = ["nat64"] nat64 = config.get_config_dict(base, key_mangling=("-", "_"), get_first_key=True) - base_src = base + ["source", "rule"] + return nat64 + + +def verify(nat64) -> None: + check_kmod(["jool"]) + base_src = ["nat64", "source", "rule"] # Load in existing instances so we can destroy any unknown lines = cmd("jool instance display --csv").splitlines() @@ -76,12 +81,8 @@ def get_config(config: Config | None = None) -> None: ): rules[num]["recreate"] = True - return nat64 - - -def verify(nat64) -> None: if not nat64: - # no need to verify the CLI as nat64 is going to be deactivated + # nothing left to do return if dict_search("source.rule", nat64): @@ -128,6 +129,9 @@ def verify(nat64) -> None: def generate(nat64) -> None: + if not nat64: + return + os.makedirs(JOOL_CONFIG_DIR, exist_ok=True) if dict_search("source.rule", nat64): @@ -184,6 +188,7 @@ def generate(nat64) -> None: def apply(nat64) -> None: if not nat64: + unload_kmod(['jool']) return if dict_search("source.rule", nat64): @@ -211,7 +216,6 @@ def apply(nat64) -> None: if __name__ == "__main__": try: - check_kmod(["jool"]) c = get_config() verify(c) generate(c) From 95eef73f1b002c8b9e8e769135ffed50c8ca6890 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 1 Aug 2024 19:18:36 -0500 Subject: [PATCH 30/64] T6629: call check_kmod within a standard config function Move the remaining calls to check_kmod within a standard function, with placement determined by the needs of the config script. --- src/conf_mode/interfaces_l2tpv3.py | 3 ++- src/conf_mode/interfaces_wireguard.py | 3 ++- src/conf_mode/interfaces_wireless.py | 3 ++- src/conf_mode/nat.py | 3 ++- src/conf_mode/nat66.py | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/conf_mode/interfaces_l2tpv3.py b/src/conf_mode/interfaces_l2tpv3.py index b9f827beea0..f0a70436e5f 100755 --- a/src/conf_mode/interfaces_l2tpv3.py +++ b/src/conf_mode/interfaces_l2tpv3.py @@ -86,6 +86,8 @@ def generate(l2tpv3): return None def apply(l2tpv3): + check_kmod(k_mod) + # Check if L2TPv3 interface already exists if interface_exists(l2tpv3['ifname']): # L2TPv3 is picky when changing tunnels/sessions, thus we can simply @@ -102,7 +104,6 @@ def apply(l2tpv3): if __name__ == '__main__': try: - check_kmod(k_mod) c = get_config() verify(c) generate(c) diff --git a/src/conf_mode/interfaces_wireguard.py b/src/conf_mode/interfaces_wireguard.py index 0e0b77877c2..482da1c6677 100755 --- a/src/conf_mode/interfaces_wireguard.py +++ b/src/conf_mode/interfaces_wireguard.py @@ -104,6 +104,8 @@ def verify(wireguard): public_keys.append(peer['public_key']) def apply(wireguard): + check_kmod('wireguard') + if 'rebuild_required' in wireguard or 'deleted' in wireguard: wg = WireGuardIf(**wireguard) # WireGuard only supports peer removal based on the configured public-key, @@ -123,7 +125,6 @@ def apply(wireguard): if __name__ == '__main__': try: - check_kmod('wireguard') c = get_config() verify(c) apply(c) diff --git a/src/conf_mode/interfaces_wireless.py b/src/conf_mode/interfaces_wireless.py index f35a250cbe9..d24675ee6ef 100755 --- a/src/conf_mode/interfaces_wireless.py +++ b/src/conf_mode/interfaces_wireless.py @@ -239,6 +239,8 @@ def verify(wifi): return None def generate(wifi): + check_kmod('mac80211') + interface = wifi['ifname'] # Delete config files if interface is removed @@ -333,7 +335,6 @@ def apply(wifi): if __name__ == '__main__': try: - check_kmod('mac80211') c = get_config() verify(c) generate(c) diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index f74bb217e50..39803fa0262 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -240,6 +240,8 @@ def generate(nat): return None def apply(nat): + check_kmod(k_mod) + cmd(f'nft --file {nftables_nat_config}') cmd(f'nft --file {nftables_static_nat_conf}') @@ -253,7 +255,6 @@ def apply(nat): if __name__ == '__main__': try: - check_kmod(k_mod) c = get_config() verify(c) generate(c) diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py index 075738dad22..c44320f36a6 100755 --- a/src/conf_mode/nat66.py +++ b/src/conf_mode/nat66.py @@ -112,6 +112,8 @@ def apply(nat): if not nat: return None + check_kmod(k_mod) + cmd(f'nft --file {nftables_nat66_config}') call_dependents() @@ -119,7 +121,6 @@ def apply(nat): if __name__ == '__main__': try: - check_kmod(k_mod) c = get_config() verify(c) generate(c) From d5ae708581d453e2205ad4cf8576503f42e262b6 Mon Sep 17 00:00:00 2001 From: fett0 Date: Fri, 2 Aug 2024 14:10:51 +0000 Subject: [PATCH 31/64] OPENVPN: T6555: fix name to bridge --- data/templates/openvpn/server.conf.j2 | 4 ++-- .../interfaces_openvpn.xml.in | 2 +- .../scripts/cli/test_interfaces_openvpn.py | 10 +++++----- src/conf_mode/interfaces_openvpn.py | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/data/templates/openvpn/server.conf.j2 b/data/templates/openvpn/server.conf.j2 index a8674ec3579..4081035582c 100644 --- a/data/templates/openvpn/server.conf.j2 +++ b/data/templates/openvpn/server.conf.j2 @@ -90,8 +90,8 @@ server-ipv6 {{ subnet }} {% endif %} {% endfor %} {% endif %} -{% if server.server_bridge is vyos_defined and server.server_bridge.disable is not vyos_defined %} -server-bridge {{ server.server_bridge.gateway }} {{ server.server_bridge.subnet_mask }} {{ server.server_bridge.start }} {{ server.server_bridge.stop if server.server_bridge.stop is vyos_defined }} +{% if server.bridge is vyos_defined and server.bridge.disable is not vyos_defined %} +server-bridge {{ server.bridge.gateway }} {{ server.bridge.subnet_mask }} {{ server.bridge.start }} {{ server.bridge.stop if server.bridge.stop is vyos_defined }} {% endif %} {% if server.client_ip_pool is vyos_defined and server.client_ip_pool.disable is not vyos_defined %} ifconfig-pool {{ server.client_ip_pool.start }} {{ server.client_ip_pool.stop }} {{ server.client_ip_pool.subnet_mask if server.client_ip_pool.subnet_mask is vyos_defined }} diff --git a/interface-definitions/interfaces_openvpn.xml.in b/interface-definitions/interfaces_openvpn.xml.in index a988bf36c7c..3563caef20d 100644 --- a/interface-definitions/interfaces_openvpn.xml.in +++ b/interface-definitions/interfaces_openvpn.xml.in @@ -445,7 +445,7 @@ - + Used with TAP device (layer 2) diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index fe04f7a20e6..5584501c5ee 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -628,7 +628,7 @@ def test_openvpn_site2site_interfaces_tun(self): def test_openvpn_server_server_bridge(self): - # Create OpenVPN server interface using server-bridge. + # Create OpenVPN server interface using bridge. # Validate configuration afterwards. br_if = 'br0' vtun_if = 'vtun5010' @@ -644,10 +644,10 @@ def test_openvpn_server_server_bridge(self): self.cli_set(path + ['encryption', 'data-ciphers', 'aes192']) self.cli_set(path + ['hash', auth_hash]) self.cli_set(path + ['mode', 'server']) - self.cli_set(path + ['server', 'server-bridge', 'gateway', gw_subnet]) - self.cli_set(path + ['server', 'server-bridge', 'start', start_subnet]) - self.cli_set(path + ['server', 'server-bridge', 'stop', stop_subnet]) - self.cli_set(path + ['server', 'server-bridge', 'subnet-mask', mask_subnet]) + self.cli_set(path + ['server', 'bridge', 'gateway', gw_subnet]) + self.cli_set(path + ['server', 'bridge', 'start', start_subnet]) + self.cli_set(path + ['server', 'bridge', 'stop', stop_subnet]) + self.cli_set(path + ['server', 'bridge', 'subnet-mask', mask_subnet]) self.cli_set(path + ['keep-alive', 'failure-count', '5']) self.cli_set(path + ['keep-alive', 'interval', '5']) self.cli_set(path + ['tls', 'ca-certificate', 'ovpn_test']) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 46bd6acf75c..95c9d3a8792 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -378,21 +378,21 @@ def verify(openvpn): if (client_v.get('ip') and len(client_v['ip']) > 1) or (client_v.get('ipv6_ip') and len(client_v['ipv6_ip']) > 1): raise ConfigError(f'Server client "{client_k}": cannot specify more than 1 IPv4 and 1 IPv6 IP') - if dict_search('server.server_bridge', openvpn): + if dict_search('server.bridge', openvpn): # check if server-bridge is a tap interfaces - if not openvpn['device_type'] == 'tap' and dict_search('server.server_bridge', openvpn): - raise ConfigError('Must specify "device-type tap" with server-bridge mode') - elif not (dict_search('server.server_bridge.start', openvpn) and dict_search('server.server_bridge.stop', openvpn)): - raise ConfigError('Server server-bridge requires both start and stop addresses') + if not openvpn['device_type'] == 'tap' and dict_search('server.bridge', openvpn): + raise ConfigError('Must specify "device-type tap" with server bridge mode') + elif not (dict_search('server.bridge.start', openvpn) and dict_search('server.bridge.stop', openvpn)): + raise ConfigError('Server server bridge requires both start and stop addresses') else: - v4PoolStart = IPv4Address(dict_search('server.server_bridge.start', openvpn)) - v4PoolStop = IPv4Address(dict_search('server.server_bridge.stop', openvpn)) + v4PoolStart = IPv4Address(dict_search('server.bridge.start', openvpn)) + v4PoolStop = IPv4Address(dict_search('server.bridge.stop', openvpn)) if v4PoolStart > v4PoolStop: - raise ConfigError(f'Server server-bridge start address {v4PoolStart} is larger than stop address {v4PoolStop}') + raise ConfigError(f'Server server bridge start address {v4PoolStart} is larger than stop address {v4PoolStop}') v4PoolSize = int(v4PoolStop) - int(v4PoolStart) if v4PoolSize >= 65536: - raise ConfigError(f'Server server_bridge is too large [{v4PoolStart} -> {v4PoolStop} = {v4PoolSize}], maximum is 65536 addresses.') + raise ConfigError(f'Server bridge is too large [{v4PoolStart} -> {v4PoolStop} = {v4PoolSize}], maximum is 65536 addresses.') if dict_search('server.client_ip_pool', openvpn): if not (dict_search('server.client_ip_pool.start', openvpn) and dict_search('server.client_ip_pool.stop', openvpn)): From 31de01242a26dff8ff993061ea2f86102a8a7493 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 1 Aug 2024 19:00:30 -0500 Subject: [PATCH 32/64] T6632: add missing standard functions to config scripts --- src/conf_mode/interfaces_wireguard.py | 3 +++ src/conf_mode/system_acceleration.py | 3 +++ src/tests/test_configd_inspect.py | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/interfaces_wireguard.py b/src/conf_mode/interfaces_wireguard.py index 0e0b77877c2..d7a638b5185 100755 --- a/src/conf_mode/interfaces_wireguard.py +++ b/src/conf_mode/interfaces_wireguard.py @@ -103,6 +103,9 @@ def verify(wireguard): public_keys.append(peer['public_key']) +def generate(wireguard): + return None + def apply(wireguard): if 'rebuild_required' in wireguard or 'deleted' in wireguard: wg = WireGuardIf(**wireguard) diff --git a/src/conf_mode/system_acceleration.py b/src/conf_mode/system_acceleration.py index e4b24867587..d2cf44ff095 100755 --- a/src/conf_mode/system_acceleration.py +++ b/src/conf_mode/system_acceleration.py @@ -79,6 +79,9 @@ def verify(qat): if not data: raise ConfigError('No QAT acceleration device found') +def generate(qat): + return + def apply(qat): # Shutdown VPN service which can use QAT if 'ipsec' in qat: diff --git a/src/tests/test_configd_inspect.py b/src/tests/test_configd_inspect.py index 98552c8f36e..ccd63189307 100644 --- a/src/tests/test_configd_inspect.py +++ b/src/tests/test_configd_inspect.py @@ -56,8 +56,7 @@ def test_signatures(self): m = import_script(s) for i in f_list: f = getattr(m, i, None) - if not f: - continue + self.assertIsNotNone(f, f"'{s}': missing function '{i}'") sig = signature(f) par = sig.parameters l = len(par) From 0162a27952d2166583a9e6aee2cd77b9c693062b Mon Sep 17 00:00:00 2001 From: fett0 Date: Fri, 2 Aug 2024 14:27:56 +0000 Subject: [PATCH 33/64] OPENVPN: T6555: fix name to bridge --- src/conf_mode/interfaces_openvpn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 95c9d3a8792..9105ce1f885 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -379,16 +379,16 @@ def verify(openvpn): raise ConfigError(f'Server client "{client_k}": cannot specify more than 1 IPv4 and 1 IPv6 IP') if dict_search('server.bridge', openvpn): - # check if server-bridge is a tap interfaces + # check if server bridge is a tap interfaces if not openvpn['device_type'] == 'tap' and dict_search('server.bridge', openvpn): raise ConfigError('Must specify "device-type tap" with server bridge mode') elif not (dict_search('server.bridge.start', openvpn) and dict_search('server.bridge.stop', openvpn)): - raise ConfigError('Server server bridge requires both start and stop addresses') + raise ConfigError('Server bridge requires both start and stop addresses') else: v4PoolStart = IPv4Address(dict_search('server.bridge.start', openvpn)) v4PoolStop = IPv4Address(dict_search('server.bridge.stop', openvpn)) if v4PoolStart > v4PoolStop: - raise ConfigError(f'Server server bridge start address {v4PoolStart} is larger than stop address {v4PoolStop}') + raise ConfigError(f'Server bridge start address {v4PoolStart} is larger than stop address {v4PoolStop}') v4PoolSize = int(v4PoolStop) - int(v4PoolStart) if v4PoolSize >= 65536: From ffbc04c591b534188cb08bf3991fadac4aa386a8 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 2 Aug 2024 18:33:17 +0300 Subject: [PATCH 34/64] T6486: generate OpenVPN use data-ciphers instead of ncp-ciphers (#3930) In the PR https://github.com/vyos/vyos-1x/pull/3823 the ncp-ciphers were replaced with `data-ciphers` fix template for "generate openvpn client-config" --- src/op_mode/generate_ovpn_client_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/op_mode/generate_ovpn_client_file.py b/src/op_mode/generate_ovpn_client_file.py index 974f7d9b68c..1d2f1067a4a 100755 --- a/src/op_mode/generate_ovpn_client_file.py +++ b/src/op_mode/generate_ovpn_client_file.py @@ -51,12 +51,12 @@ } %} {% if encryption is defined and encryption is not none %} -{% if encryption.ncp_ciphers is defined and encryption.ncp_ciphers is not none %} -cipher {% for algo in encryption.ncp_ciphers %} +{% if encryption.data_ciphers is defined and encryption.data_ciphers is not none %} +cipher {% for algo in encryption.data_ciphers %} {{ encryption_map[algo] if algo in encryption_map.keys() else algo }}{% if not loop.last %}:{% endif %} {% endfor %} -data-ciphers {% for algo in encryption.ncp_ciphers %} +data-ciphers {% for algo in encryption.data_ciphers %} {{ encryption_map[algo] if algo in encryption_map.keys() else algo }}{% if not loop.last %}:{% endif %} {% endfor %} {% endif %} From f2256ad338fc3fbaa9a5de2c0615603cd23e0f94 Mon Sep 17 00:00:00 2001 From: Roman Khramshin Date: Fri, 2 Aug 2024 21:41:05 +0600 Subject: [PATCH 35/64] T6619: Remove the remaining uses of per-protocol FRR configs (#3916) --- data/templates/frr/static_mcast.frr.j2 | 9 ---- .../cli/test_protocols_static_multicast.py | 49 +++++++++++++++++++ src/conf_mode/protocols_static_multicast.py | 27 ++++++++-- 3 files changed, 71 insertions(+), 14 deletions(-) create mode 100755 smoketest/scripts/cli/test_protocols_static_multicast.py diff --git a/data/templates/frr/static_mcast.frr.j2 b/data/templates/frr/static_mcast.frr.j2 index 491d4b54a10..54b2790b07f 100644 --- a/data/templates/frr/static_mcast.frr.j2 +++ b/data/templates/frr/static_mcast.frr.j2 @@ -1,13 +1,4 @@ ! -{% for route_gr in old_mroute %} -{% for nh in old_mroute[route_gr] %} -{% if old_mroute[route_gr][nh] %} -no ip mroute {{ route_gr }} {{ nh }} {{ old_mroute[route_gr][nh] }} -{% else %} -no ip mroute {{ route_gr }} {{ nh }} -{% endif %} -{% endfor %} -{% endfor %} {% for route_gr in mroute %} {% for nh in mroute[route_gr] %} {% if mroute[route_gr][nh] %} diff --git a/smoketest/scripts/cli/test_protocols_static_multicast.py b/smoketest/scripts/cli/test_protocols_static_multicast.py new file mode 100755 index 00000000000..9fdda236f6b --- /dev/null +++ b/smoketest/scripts/cli/test_protocols_static_multicast.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest + +from base_vyostest_shim import VyOSUnitTestSHIM + + +base_path = ['protocols', 'static', 'multicast'] + + +class TestProtocolsStaticMulticast(VyOSUnitTestSHIM.TestCase): + + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + mroute = self.getFRRconfig('ip mroute', end='') + self.assertFalse(mroute) + + def test_01_static_multicast(self): + + self.cli_set(base_path + ['route', '224.202.0.0/24', 'next-hop', '224.203.0.1']) + self.cli_set(base_path + ['interface-route', '224.203.0.0/24', 'next-hop-interface', 'eth0']) + + self.cli_commit() + + # Verify FRR bgpd configuration + frrconfig = self.getFRRconfig('ip mroute', end='') + + self.assertIn('ip mroute 224.202.0.0/24 224.203.0.1', frrconfig) + self.assertIn('ip mroute 224.203.0.0/24 eth0', frrconfig) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/conf_mode/protocols_static_multicast.py b/src/conf_mode/protocols_static_multicast.py index 7f6ae3680bc..2bf79404202 100755 --- a/src/conf_mode/protocols_static_multicast.py +++ b/src/conf_mode/protocols_static_multicast.py @@ -20,9 +20,10 @@ from sys import exit from vyos import ConfigError +from vyos import frr from vyos.config import Config from vyos.utils.process import call -from vyos.template import render +from vyos.template import render, render_to_string from vyos import airbag airbag.enable() @@ -92,23 +93,39 @@ def verify(mroute): if IPv4Address(route[0]) < IPv4Address('224.0.0.0'): raise ConfigError(route + " not a multicast network") + def generate(mroute): if mroute is None: return None - render(config_file, 'frr/static_mcast.frr.j2', mroute) + mroute['new_frr_config'] = render_to_string('frr/static_mcast.frr.j2', mroute) return None + def apply(mroute): if mroute is None: return None + static_daemon = 'staticd' + + frr_cfg = frr.FRRConfig() + frr_cfg.load_configuration(static_daemon) - if os.path.exists(config_file): - call(f'vtysh -d staticd -f {config_file}') - os.remove(config_file) + if 'old_mroute' in mroute: + for route_gr in mroute['old_mroute']: + for nh in mroute['old_mroute'][route_gr]: + if mroute['old_mroute'][route_gr][nh]: + frr_cfg.modify_section(f'^ip mroute {route_gr} {nh} {mroute["old_mroute"][route_gr][nh]}') + else: + frr_cfg.modify_section(f'^ip mroute {route_gr} {nh}') + + if 'new_frr_config' in mroute: + frr_cfg.add_before(frr.default_add_before, mroute['new_frr_config']) + + frr_cfg.commit_configuration(static_daemon) return None + if __name__ == '__main__': try: c = get_config() From 9979afa15650bd609399030da1751488baaac70b Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 4 Aug 2024 08:45:12 +0200 Subject: [PATCH 36/64] multicast: T6619: remove unused imports --- src/conf_mode/protocols_static_multicast.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/conf_mode/protocols_static_multicast.py b/src/conf_mode/protocols_static_multicast.py index 2bf79404202..d323ceb4f94 100755 --- a/src/conf_mode/protocols_static_multicast.py +++ b/src/conf_mode/protocols_static_multicast.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020 VyOS maintainers and contributors +# Copyright (C) 2020-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -14,7 +14,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os from ipaddress import IPv4Address from sys import exit @@ -22,8 +21,7 @@ from vyos import ConfigError from vyos import frr from vyos.config import Config -from vyos.utils.process import call -from vyos.template import render, render_to_string +from vyos.template import render_to_string from vyos import airbag airbag.enable() From 4d2b10ff1f4e862abf1c11f3cfbdc2a910c48301 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 4 Aug 2024 08:45:43 +0200 Subject: [PATCH 37/64] ipsec: T5873: remove unused imports --- src/conf_mode/vpn_ipsec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index e8a0bc41473..b3e05a814a6 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -25,7 +25,6 @@ from netaddr import IPNetwork from netaddr import IPRange -from vyos.base import Warning from vyos.config import Config from vyos.config import config_dict_merge from vyos.configdep import set_dependents From 60b0614296874c144665417130d4881461114db0 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Sun, 4 Aug 2024 17:52:57 +1000 Subject: [PATCH 38/64] firewall: T4694: Adding GRE flags & fields matches to firewall rules * Only matching flags and fields used by modern RFC2890 "extended GRE" - this is backwards-compatible, but does not match all possible flags. * There are no nftables helpers for the GRE key field, which is critical to match individual tunnel sessions (more detail in the forum post) * nft expression syntax is not flexible enough for multiple field matches in a single rule and the key offset changes depending on flags. * Thus, clumsy compromise in requiring an explicit match on the "checksum" flag if a key is present, so we know where key will be. In most cases, nobody uses the checksum, but assuming it to be off or automatically adding a "not checksum" match unless told otherwise would be confusing * The automatic "flags key" check when specifying a key doesn't have similar validation, I added it first and it makes sense. I would still like to find a workaround to the "checksum" offset problem. * If we could add 2 rules from 1 config definition, we could match both cases with appropriate offsets, but this would break existing FW generation logic, logging, etc. * Added a "test_gre_match" smoketest --- .../include/firewall/common-rule-inet.xml.i | 1 + .../include/firewall/gre.xml.i | 116 ++++++++++++++++++ python/vyos/firewall.py | 61 +++++++++ smoketest/scripts/cli/test_firewall.py | 55 +++++++++ src/conf_mode/firewall.py | 30 +++++ 5 files changed, 263 insertions(+) create mode 100644 interface-definitions/include/firewall/gre.xml.i diff --git a/interface-definitions/include/firewall/common-rule-inet.xml.i b/interface-definitions/include/firewall/common-rule-inet.xml.i index 0acb08ec9b0..e44938b14b3 100644 --- a/interface-definitions/include/firewall/common-rule-inet.xml.i +++ b/interface-definitions/include/firewall/common-rule-inet.xml.i @@ -19,5 +19,6 @@ #include #include #include +#include #include diff --git a/interface-definitions/include/firewall/gre.xml.i b/interface-definitions/include/firewall/gre.xml.i new file mode 100644 index 00000000000..2852334347a --- /dev/null +++ b/interface-definitions/include/firewall/gre.xml.i @@ -0,0 +1,116 @@ + + + + GRE fields to match + + + + + GRE flag bits to match + + + + + Header includes optional key field + + + + + Header does not include optional key field + + + + + + + + Header includes optional checksum + + + + + Header does not include optional checksum + + + + + + + + Header includes a sequence number field + + + + + Header does not include a sequence number field + + + + + + + + + + EtherType of encapsulated packet + + ip ip6 arp 802.1q 802.1ad + + + u32:0-65535 + Ethernet protocol number + + + u32:0x0-0xffff + Ethernet protocol number (hex) + + + ip + IPv4 + + + ip6 + IPv6 + + + arp + Address Resolution Protocol + + + 802.1q + VLAN-tagged frames (IEEE 802.1q) + + + 802.1ad + Provider Bridging (IEEE 802.1ad, Q-in-Q) + + + gretap + Transparent Ethernet Bridging (L2 Ethernet over GRE, gretap) + + + (ip|ip6|arp|802.1q|802.1ad|gretap|0x[0-9a-fA-F]{1,4}) + + + + + #include + + + GRE Version + + gre + Standard GRE + + + pptp + Point to Point Tunnelling Protocol + + + (gre|pptp) + + + + + + diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index cac6d2953cb..3976a5580b8 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -409,6 +409,41 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): time = rule_conf['recent']['time'] output.append(f'add @RECENT{def_suffix}_{hook}_{fw_name}_{rule_id} {{ {ip_name} saddr limit rate over {count}/{time} burst {count} packets }}') + if 'gre' in rule_conf: + gre_key = dict_search_args(rule_conf, 'gre', 'key') + + gre_flags = dict_search_args(rule_conf, 'gre', 'flags') + output.append(parse_gre_flags(gre_flags or {}, force_keyed=gre_key is not None)) + + gre_proto_alias_map = { + '802.1q': '8021q', + '802.1ad': '8021ad', + 'gretap': '0x6558', + } + + gre_proto = dict_search_args(rule_conf, 'gre', 'inner_proto') + if gre_proto is not None: + gre_proto = gre_proto_alias_map.get(gre_proto, gre_proto) + output.append(f'gre protocol {gre_proto}') + + gre_ver = dict_search_args(rule_conf, 'gre', 'version') + if gre_ver == 'gre': + output.append('gre version 0') + elif gre_ver == 'pptp': + output.append('gre version 1') + + if gre_key: + # The offset of the key within the packet shifts depending on the C-flag. + # nftables cannot handle complex enough expressions to match multiple + # offsets based on bitfields elsewhere. + # We enforce a specific match for the checksum flag in validation, so the + # gre_flags dict will always have a 'checksum' key when gre_key is populated. + if not gre_flags['checksum']: + # No "unset" child node means C is set, we offset key lookup +32 bits + output.append(f'@th,64,32 == {gre_key}') + else: + output.append(f'@th,32,32 == {gre_key}') + if 'time' in rule_conf: output.append(parse_time(rule_conf['time'])) @@ -544,6 +579,32 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): output.append(f'comment "{family}-{hook}-{fw_name}-{rule_id}"') return " ".join(output) +def parse_gre_flags(flags, force_keyed=False): + flag_map = { # nft does not have symbolic names for these. + 'checksum': 1<<0, + 'routing': 1<<1, + 'key': 1<<2, + 'sequence': 1<<3, + 'strict_routing': 1<<4, + } + + include = 0 + exclude = 0 + for fl_name, fl_state in flags.items(): + if not fl_state: + include |= flag_map[fl_name] + else: # 'unset' child tag + exclude |= flag_map[fl_name] + + if force_keyed: + # Implied by a key-match. + include |= flag_map['key'] + + if include == 0 and exclude == 0: + return '' # Don't bother extracting and matching no bits + + return f'gre flags & {include + exclude} == {include}' + def parse_tcp_flags(flags): include = [flag for flag in flags if flag != 'not'] exclude = list(flags['not']) if 'not' in flags else [] diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 551f8ce654f..dfc816a4206 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -1100,5 +1100,60 @@ def test_cyclic_jump_validation(self): with self.assertRaises(ConfigSessionError): self.cli_commit() + def test_gre_match(self): + name = 'smoketest-gre' + + self.cli_set(['firewall', 'ipv4', 'name', name, 'default-action', 'return']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'action', 'accept']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'protocol', 'gre']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'gre', 'flags', 'key']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'gre', 'flags', 'checksum', 'unset']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'gre', 'key', '1234']) + self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '1', 'log']) + + self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '2', 'action', 'continue']) + self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '2', 'protocol', 'gre']) + self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '2', 'gre', 'inner-proto', '0x6558']) + self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '2', 'log']) + + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'protocol', 'gre']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'gre', 'flags', 'checksum']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'gre', 'key', '4321']) + + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '4', 'action', 'reject']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '4', 'protocol', 'gre']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '4', 'gre', 'version', 'pptp']) + + self.cli_commit() + + nftables_search_v4 = [ + ['gre protocol 0x6558', 'continue comment'], + ['gre flags & 5 == 4 @th,32,32 0x4d2', 'accept comment'], + ] + + nftables_search_v6 = [ + ['gre flags & 5 == 5 @th,64,32 0x10e1', 'drop comment'], + ['gre version 1', 'reject comment'], + ] + + self.verify_nftables(nftables_search_v4, 'ip vyos_filter') + self.verify_nftables(nftables_search_v6, 'ip6 vyos_filter') + + # GRE match will only work with protocol GRE + self.cli_delete(['firewall', 'ipv4', 'name', name, 'rule', '1', 'protocol', 'gre']) + + with self.assertRaises(ConfigSessionError): + self.cli_commit() + + self.cli_discard() + + # GREv1 (PPTP) does not include a key field, match not available + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '4', 'gre', 'flags', 'checksum', 'unset']) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '4', 'gre', 'key', '1234']) + + with self.assertRaises(ConfigSessionError): + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 02bf00bcc1f..b71ce7124ff 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -231,6 +231,36 @@ def verify_rule(firewall, family, hook, priority, rule_id, rule_conf): if not {'count', 'time'} <= set(rule_conf['recent']): raise ConfigError('Recent "count" and "time" values must be defined') + if 'gre' in rule_conf: + if dict_search_args(rule_conf, 'protocol') != 'gre': + raise ConfigError('Protocol must be gre when matching GRE flags and fields') + + if dict_search_args(rule_conf, 'gre', 'key'): + if dict_search_args(rule_conf, 'gre', 'version') == 'pptp': + raise ConfigError('GRE tunnel keys are not present in PPTP') + + if dict_search_args(rule_conf, 'gre', 'flags', 'checksum') is None: + # There is no builtin match in nftables for the GRE key, so we need to do a raw lookup. + # The offset of the key within the packet shifts depending on the C-flag. + # 99% of the time, nobody will have checksums enabled - it's usually a manual config option. + # We can either assume it is unset unless otherwise directed + # (confusing, requires doco to explain why it doesn't work sometimes) + # or, demand an explicit selection to be made for this specific match rule. + # This check enforces the latter. The user is free to create rules for both cases. + raise ConfigError('Matching GRE tunnel key requires an explicit checksum flag match. For most cases, use "gre flags checksum unset"') + + if dict_search_args(rule_conf, 'gre', 'flags', 'key', 'unset') is not None: + raise ConfigError('Matching GRE tunnel key implies "flags key", cannot specify "flags key unset"') + + gre_inner_proto = dict_search_args(rule_conf, 'gre', 'inner_proto') + if gre_inner_proto is not None: + try: + gre_inner_value = int(gre_inner_proto, 0) + if gre_inner_value < 0 or gre_inner_value > 65535: + raise ConfigError('inner-proto outside valid ethertype range 0-65535') + except ValueError: + pass # Symbolic constant, pre-validated before reaching here. + tcp_flags = dict_search_args(rule_conf, 'tcp', 'flags') if tcp_flags: if dict_search_args(rule_conf, 'protocol') != 'tcp': From d08f06ac7d13da95d311497d0c4486e66b721016 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 5 Aug 2024 05:44:12 +0000 Subject: [PATCH 39/64] GitHub: T6560: checkout pull request HEAD commit instead of merge commit --- .github/workflows/package-smoketest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/package-smoketest.yml b/.github/workflows/package-smoketest.yml index 0a8208b8718..467ff062ef7 100644 --- a/.github/workflows/package-smoketest.yml +++ b/.github/workflows/package-smoketest.yml @@ -37,6 +37,8 @@ jobs: uses: actions/checkout@v4 with: path: packages/vyos-1x + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} - name: Build vyos-1x package run: | cd packages/vyos-1x; dpkg-buildpackage -uc -us -tc -b From 6e910723ed9bd7f510f39379298bd98375608bfa Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 5 Aug 2024 05:44:29 +0000 Subject: [PATCH 40/64] firewall: T4694: fix GRE key include path in XML --- interface-definitions/include/firewall/gre.xml.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface-definitions/include/firewall/gre.xml.i b/interface-definitions/include/firewall/gre.xml.i index 2852334347a..e7b9fd5b194 100644 --- a/interface-definitions/include/firewall/gre.xml.i +++ b/interface-definitions/include/firewall/gre.xml.i @@ -33,7 +33,7 @@ - + @@ -94,7 +94,7 @@ - #include + #include GRE Version From 2fd817e51532c6428c95704233e62585e76b2ad8 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 5 Aug 2024 05:45:00 +0000 Subject: [PATCH 41/64] smoketest: T6555: openvpn: SyntaxError: '(' was never closed --- smoketest/scripts/cli/test_interfaces_openvpn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index 5584501c5ee..f23a28c2599 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -678,7 +678,7 @@ def test_openvpn_server_server_bridge(self): self.assertIn(f'dh /run/openvpn/{vtun_if}_dh.pem', config) # check that no interface remained after deleting them - self.cli_delete((['interfaces', 'bridge', br_if, 'member', 'interface', vtun_if]) + self.cli_delete(['interfaces', 'bridge', br_if, 'member', 'interface', vtun_if]) self.cli_delete(base_path) self.cli_commit() From 9bd2c196fe238a38f4fd0977efd1727333e7770e Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 5 Aug 2024 20:22:25 +0200 Subject: [PATCH 42/64] smoketest: T6555: openvpn: NameError: name 'elf' is not defined --- smoketest/scripts/cli/test_interfaces_openvpn.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py index f23a28c2599..d8a091aaafa 100755 --- a/smoketest/scripts/cli/test_interfaces_openvpn.py +++ b/smoketest/scripts/cli/test_interfaces_openvpn.py @@ -656,8 +656,6 @@ def test_openvpn_server_server_bridge(self): self.cli_commit() - - config_file = f'/run/openvpn/{vtun_if}.conf' config = read_file(config_file) self.assertIn(f'dev {vtun_if}', config) @@ -667,9 +665,7 @@ def test_openvpn_server_server_bridge(self): self.assertIn(f'data-ciphers AES-192-CBC', config) self.assertIn(f'mode server', config) self.assertIn(f'server-bridge {gw_subnet} {mask_subnet} {start_subnet} {stop_subnet}', config) - elf.assertIn(f'keepalive 5 25', config) - - + self.assertIn(f'keepalive 5 25', config) # TLS options self.assertIn(f'ca /run/openvpn/{vtun_if}_ca.pem', config) From 8500e8658ff10f52739143fd7814cf60c9195f16 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 5 Aug 2024 17:09:07 +0200 Subject: [PATCH 43/64] sysctl: T3204: restore sysctl setttings overwritten by tuned --- data/config-mode-dependencies/vyos-1x.json | 10 ++- smoketest/scripts/cli/test_system_option.py | 84 +++++++++++++++++++++ src/conf_mode/system_ip.py | 10 ++- src/conf_mode/system_ipv6.py | 9 +++ src/conf_mode/system_option.py | 15 ++-- 5 files changed, 118 insertions(+), 10 deletions(-) create mode 100755 smoketest/scripts/cli/test_system_option.py diff --git a/data/config-mode-dependencies/vyos-1x.json b/data/config-mode-dependencies/vyos-1x.json index 2398425507a..2981a0851f0 100644 --- a/data/config-mode-dependencies/vyos-1x.json +++ b/data/config-mode-dependencies/vyos-1x.json @@ -64,8 +64,14 @@ "system_wireless": { "wireless": ["interfaces_wireless"] }, + "system_ip": { + "sysctl": ["system_sysctl"] + }, + "system_ipv6": { + "sysctl": ["system_sysctl"] + }, "system_option": { - "ip": ["system_ip"], - "ipv6": ["system_ipv6"] + "ip_ipv6": ["system_ip", "system_ipv6"], + "sysctl": ["system_sysctl"] } } diff --git a/smoketest/scripts/cli/test_system_option.py b/smoketest/scripts/cli/test_system_option.py new file mode 100755 index 00000000000..c6f48bfc692 --- /dev/null +++ b/smoketest/scripts/cli/test_system_option.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import unittest +from base_vyostest_shim import VyOSUnitTestSHIM +from vyos.utils.file import read_file +from vyos.utils.process import is_systemd_service_active +from vyos.utils.system import sysctl_read + +base_path = ['system', 'option'] + +class TestSystemOption(VyOSUnitTestSHIM.TestCase): + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + def test_ctrl_alt_delete(self): + self.cli_set(base_path + ['ctrl-alt-delete', 'reboot']) + self.cli_commit() + + tmp = os.readlink('/lib/systemd/system/ctrl-alt-del.target') + self.assertEqual(tmp, '/lib/systemd/system/reboot.target') + + self.cli_set(base_path + ['ctrl-alt-delete', 'poweroff']) + self.cli_commit() + + tmp = os.readlink('/lib/systemd/system/ctrl-alt-del.target') + self.assertEqual(tmp, '/lib/systemd/system/poweroff.target') + + self.cli_delete(base_path + ['ctrl-alt-delete', 'poweroff']) + self.cli_commit() + self.assertFalse(os.path.exists('/lib/systemd/system/ctrl-alt-del.target')) + + def test_reboot_on_panic(self): + panic_file = '/proc/sys/kernel/panic' + + tmp = read_file(panic_file) + self.assertEqual(tmp, '0') + + self.cli_set(base_path + ['reboot-on-panic']) + self.cli_commit() + + tmp = read_file(panic_file) + self.assertEqual(tmp, '60') + + def test_performance(self): + tuned_service = 'tuned.service' + + self.assertFalse(is_systemd_service_active(tuned_service)) + + # T3204 sysctl options must not be overwritten by tuned + gc_thresh1 = '131072' + gc_thresh2 = '262000' + gc_thresh3 = '524000' + + self.cli_set(['system', 'sysctl', 'parameter', 'net.ipv4.neigh.default.gc_thresh1', 'value', gc_thresh1]) + self.cli_set(['system', 'sysctl', 'parameter', 'net.ipv4.neigh.default.gc_thresh2', 'value', gc_thresh2]) + self.cli_set(['system', 'sysctl', 'parameter', 'net.ipv4.neigh.default.gc_thresh3', 'value', gc_thresh3]) + + self.cli_set(base_path + ['performance', 'throughput']) + self.cli_commit() + + self.assertTrue(is_systemd_service_active(tuned_service)) + + self.assertEqual(sysctl_read('net.ipv4.neigh.default.gc_thresh1'), gc_thresh1) + self.assertEqual(sysctl_read('net.ipv4.neigh.default.gc_thresh2'), gc_thresh2) + self.assertEqual(sysctl_read('net.ipv4.neigh.default.gc_thresh3'), gc_thresh3) + +if __name__ == '__main__': + unittest.main(verbosity=2, failfast=True) diff --git a/src/conf_mode/system_ip.py b/src/conf_mode/system_ip.py index 2a0bda91a34..c8a91fd2f10 100755 --- a/src/conf_mode/system_ip.py +++ b/src/conf_mode/system_ip.py @@ -24,7 +24,8 @@ from vyos.utils.file import write_file from vyos.utils.process import is_systemd_service_active from vyos.utils.system import sysctl_write - +from vyos.configdep import set_dependents +from vyos.configdep import call_dependents from vyos import ConfigError from vyos import frr from vyos import airbag @@ -52,6 +53,11 @@ def get_config(config=None): get_first_key=True)}} # Merge policy dict into "regular" config dict opt = dict_merge(tmp, opt) + + # If IPv4 ARP table size is set here and also manually in sysctl, the more + # fine grained value from sysctl must win + set_dependents('sysctl', conf) + return opt def verify(opt): @@ -127,6 +133,8 @@ def apply(opt): frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config']) frr_cfg.commit_configuration(zebra_daemon) + call_dependents() + if __name__ == '__main__': try: c = get_config() diff --git a/src/conf_mode/system_ipv6.py b/src/conf_mode/system_ipv6.py index 00d440e35e4..a2442d0099e 100755 --- a/src/conf_mode/system_ipv6.py +++ b/src/conf_mode/system_ipv6.py @@ -25,6 +25,8 @@ from vyos.utils.file import write_file from vyos.utils.process import is_systemd_service_active from vyos.utils.system import sysctl_write +from vyos.configdep import set_dependents +from vyos.configdep import call_dependents from vyos import ConfigError from vyos import frr from vyos import airbag @@ -52,6 +54,11 @@ def get_config(config=None): get_first_key=True)}} # Merge policy dict into "regular" config dict opt = dict_merge(tmp, opt) + + # If IPv6 neighbor table size is set here and also manually in sysctl, the more + # fine grained value from sysctl must win + set_dependents('sysctl', conf) + return opt def verify(opt): @@ -110,6 +117,8 @@ def apply(opt): frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config']) frr_cfg.commit_configuration(zebra_daemon) + call_dependents() + if __name__ == '__main__': try: c = get_config() diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py index 18068692469..4025104924d 100755 --- a/src/conf_mode/system_option.py +++ b/src/conf_mode/system_option.py @@ -31,7 +31,8 @@ from vyos.utils.process import is_systemd_service_running from vyos.utils.network import is_addr_assigned from vyos.utils.network import is_intf_addr_assigned -from vyos.configdep import set_dependents, call_dependents +from vyos.configdep import set_dependents +from vyos.configdep import call_dependents from vyos import ConfigError from vyos import airbag airbag.enable() @@ -57,10 +58,9 @@ def get_config(config=None): with_recursive_defaults=True) if 'performance' in options: - # Update IPv4 and IPv6 options after TuneD reapplies - # sysctl from config files - for protocol in ['ip', 'ipv6']: - set_dependents(protocol, conf) + # Update IPv4/IPv6 and sysctl options after tuned applied it's settings + set_dependents('ip_ipv6', conf) + set_dependents('sysctl', conf) return options @@ -111,10 +111,11 @@ def generate(options): def apply(options): # System bootup beep + beep_service = 'vyos-beep.service' if 'startup_beep' in options: - cmd('systemctl enable vyos-beep.service') + cmd(f'systemctl enable {beep_service}') else: - cmd('systemctl disable vyos-beep.service') + cmd(f'systemctl disable {beep_service}') # Ctrl-Alt-Delete action if os.path.exists(systemd_action_file): From acd27ebb5f80d7356d727742260d6c01d4ebce7a Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 5 Aug 2024 22:57:51 +0300 Subject: [PATCH 44/64] T6634: README: Add image graphs of contributors (#3944) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 255d7784622..4b7d7adb940 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,9 @@ Runtime tests are executed by the CI system on a running VyOS instance inside QEMU. The testcases can be found inside the smoketest subdirectory which will be placed into the vyos-1x-smoketest package. + +### Thanks to all the people who already contributed! + + + + From b67786df8067deb34f66e8a5c2bba66745b8445e Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:30:50 +0530 Subject: [PATCH 45/64] T6637: py files filter added for unused import check --- .github/workflows/check-unused-imports.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml index 76bba94be80..9fca5add6f4 100644 --- a/.github/workflows/check-unused-imports.yml +++ b/.github/workflows/check-unused-imports.yml @@ -1,13 +1,8 @@ name: Check for unused imports using Pylint on: - pull_request_target: + pull_request: branches: - current - paths: - - '**' - - '!.github/**' - - '!**/*.md' - workflow_dispatch: permissions: pull-requests: write From c4ffc87da9a1fc9655df189a31797cf31a94e41a Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 6 Aug 2024 11:55:08 +0200 Subject: [PATCH 46/64] smoketest: T6614: add op-mode test for Kernel version (#3946) --- smoketest/scripts/cli/test_op-mode_show.py | 7 +++++++ smoketest/scripts/cli/test_service_stunnel.py | 0 2 files changed, 7 insertions(+) mode change 100644 => 100755 smoketest/scripts/cli/test_service_stunnel.py diff --git a/smoketest/scripts/cli/test_op-mode_show.py b/smoketest/scripts/cli/test_op-mode_show.py index fba60cc0160..62f8e88dac0 100755 --- a/smoketest/scripts/cli/test_op-mode_show.py +++ b/smoketest/scripts/cli/test_op-mode_show.py @@ -15,8 +15,10 @@ # along with this program. If not, see . import unittest + from base_vyostest_shim import VyOSUnitTestSHIM +from vyos.utils.process import cmd from vyos.version import get_version base_path = ['show'] @@ -29,6 +31,11 @@ def test_op_mode_show_version(self): version = get_version() self.assertIn(f'Version: VyOS {version}', tmp) + def test_op_mode_show_version_kernel(self): + # Retrieve output of "show version" OP-mode command + tmp = self.op_mode(base_path + ['version', 'kernel']) + self.assertEqual(cmd('uname -r'), tmp) + def test_op_mode_show_vrf(self): # Retrieve output of "show version" OP-mode command tmp = self.op_mode(base_path + ['vrf']) diff --git a/smoketest/scripts/cli/test_service_stunnel.py b/smoketest/scripts/cli/test_service_stunnel.py old mode 100644 new mode 100755 From 6543f444c42ff45e8115366256643186bf1dd567 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 6 Aug 2024 20:21:12 -0500 Subject: [PATCH 47/64] configd: T6640: enforce in_session returns False under configd The CStore in_session check is a false positive outside of a config session if a specific environment variable is set with an existing referent in unionfs. To allow extensions when running under configd and avoid confusion, enforce in_session returns False. --- python/vyos/configsource.py | 2 ++ src/services/vyos-configd | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index f582bdfab0e..59e5ac8a19b 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -204,6 +204,8 @@ def in_session(self): Returns: True if called from a configuration session, False otherwise. """ + if os.getenv('VYOS_CONFIGD', ''): + return False try: self._run(self._make_command('inSession', '')) return True diff --git a/src/services/vyos-configd b/src/services/vyos-configd index a4b839a7fdc..69ee15bf18b 100755 --- a/src/services/vyos-configd +++ b/src/services/vyos-configd @@ -267,6 +267,8 @@ if __name__ == '__main__': cfg_group = grp.getgrnam(CFG_GROUP) os.setgid(cfg_group.gr_gid) + os.environ['VYOS_CONFIGD'] = 't' + def sig_handler(signum, frame): shutdown() From ed63c9d1896a218715e13e1799fc059f4561f75e Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 8 Aug 2024 11:24:00 -0500 Subject: [PATCH 48/64] qos: T6638: require interface state existence in verify conditional --- python/vyos/configverify.py | 13 +++++++------ src/conf_mode/qos.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 4cb84194aa0..b49d66c36eb 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -237,7 +237,7 @@ def verify_bridge_delete(config): raise ConfigError(f'Interface "{interface}" cannot be deleted as it ' f'is a member of bridge "{bridge_name}"!') -def verify_interface_exists(ifname, warning_only=False): +def verify_interface_exists(ifname, state_required=False, warning_only=False): """ Common helper function used by interface implementations to perform recurring validation if an interface actually exists. We first probe @@ -249,11 +249,12 @@ def verify_interface_exists(ifname, warning_only=False): from vyos.utils.dict import dict_search_recursive from vyos.utils.network import interface_exists - # Check if interface is present in CLI config - config = ConfigTreeQuery() - tmp = config.get_config_dict(['interfaces'], get_first_key=True) - if bool(list(dict_search_recursive(tmp, ifname))): - return True + if not state_required: + # Check if interface is present in CLI config + config = ConfigTreeQuery() + tmp = config.get_config_dict(['interfaces'], get_first_key=True) + if bool(list(dict_search_recursive(tmp, ifname))): + return True # Interface not found on CLI, try Linux Kernel if interface_exists(ifname): diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 45248fb4a93..464d7c1920a 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -303,7 +303,7 @@ def apply(qos): return None for interface, interface_config in qos['interface'].items(): - if not verify_interface_exists(interface, warning_only=True): + if not verify_interface_exists(interface, state_required=True, warning_only=True): # When shaper is bound to a dialup (e.g. PPPoE) interface it is # possible that it is yet not availbale when to QoS code runs. # Skip the configuration and inform the user via warning_only=True From ff58f3e5f30d3775487a6a3b561863aa37d11d43 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Fri, 9 Aug 2024 14:03:21 +0000 Subject: [PATCH 49/64] T6643: firewall: fix ip address range parsing on firewall rules. --- python/vyos/firewall.py | 15 ++++++++++++--- smoketest/scripts/cli/test_firewall.py | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) mode change 100644 => 100755 python/vyos/firewall.py diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py old mode 100644 new mode 100755 index 3976a5580b8..f0cf3c924be --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -167,10 +167,19 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if address_mask: operator = '!=' if exclude else '==' operator = f'& {address_mask} {operator} ' - if is_ipv4(suffix): - output.append(f'ip {prefix}addr {operator}{suffix}') + + if suffix.find('-') != -1: + # Range + start, end = suffix.split('-') + if is_ipv4(start): + output.append(f'ip {prefix}addr {operator}{suffix}') + else: + output.append(f'ip6 {prefix}addr {operator}{suffix}') else: - output.append(f'ip6 {prefix}addr {operator}{suffix}') + if is_ipv4(suffix): + output.append(f'ip {prefix}addr {operator}{suffix}') + else: + output.append(f'ip6 {prefix}addr {operator}{suffix}') if 'fqdn' in side_conf: fqdn = side_conf['fqdn'] diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index dfc816a4206..8aeeff1496b 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -311,7 +311,7 @@ def test_ipv4_advanced(self): self.cli_set(['firewall', 'ipv4', 'name', name, 'rule', '7', 'dscp-exclude', '21-25']) self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'default-action', 'drop']) - self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'source', 'address', '198.51.100.1']) + self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'source', 'address', '198.51.100.1-198.51.100.50']) self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'mark', '1010']) self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'action', 'jump']) self.cli_set(['firewall', 'ipv4', 'forward', 'filter', 'rule', '1', 'jump-target', name]) @@ -331,7 +331,7 @@ def test_ipv4_advanced(self): nftables_search = [ ['chain VYOS_FORWARD_filter'], ['type filter hook forward priority filter; policy accept;'], - ['ip saddr 198.51.100.1', 'meta mark 0x000003f2', f'jump NAME_{name}'], + ['ip saddr 198.51.100.1-198.51.100.50', 'meta mark 0x000003f2', f'jump NAME_{name}'], ['FWD-filter default-action drop', 'drop'], ['chain VYOS_INPUT_filter'], ['type filter hook input priority filter; policy accept;'], @@ -455,7 +455,7 @@ def test_ipv6_basic_rules(self): self.cli_set(['firewall', 'ipv6', 'name', name, 'default-log']) self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'action', 'accept']) - self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'source', 'address', '2002::1']) + self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'source', 'address', '2002::1-2002::10']) self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'destination', 'address', '2002::1:1']) self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'log']) self.cli_set(['firewall', 'ipv6', 'name', name, 'rule', '1', 'log-options', 'level', 'crit']) @@ -510,7 +510,7 @@ def test_ipv6_basic_rules(self): ['tcp dport 23', 'drop'], ['PRE-raw default-action accept', 'accept'], [f'chain NAME6_{name}'], - ['saddr 2002::1', 'daddr 2002::1:1', 'log prefix "[ipv6-NAM-v6-smoketest-1-A]" log level crit', 'accept'], + ['saddr 2002::1-2002::10', 'daddr 2002::1:1', 'log prefix "[ipv6-NAM-v6-smoketest-1-A]" log level crit', 'accept'], [f'"{name} default-action drop"', f'log prefix "[ipv6-{name}-default-D]"', 'drop'], ['jump VYOS_STATE_POLICY6'], ['chain VYOS_STATE_POLICY6'], From 9c442f51569f1cb9de552973ba7cec19248692c4 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Sat, 10 Aug 2024 01:22:40 +0530 Subject: [PATCH 50/64] T6637: add pr commenting back in un-used import check --- .github/workflows/check-unused-imports.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml index 9fca5add6f4..d6dd614834c 100644 --- a/.github/workflows/check-unused-imports.yml +++ b/.github/workflows/check-unused-imports.yml @@ -1,6 +1,6 @@ name: Check for unused imports using Pylint on: - pull_request: + pull_request_target: branches: - current From 832056ad5171b28c50270389c4537f6c7a542d59 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 4 Aug 2024 21:19:32 -0500 Subject: [PATCH 51/64] xml: T6650: add initial op-mode cache support --- .gitignore | 2 + Makefile | 2 + python/vyos/defaults.py | 3 +- python/vyos/xml_ref/__init__.py | 23 +++ python/vyos/xml_ref/generate_op_cache.py | 176 +++++++++++++++++++++++ python/vyos/xml_ref/op_definition.py | 49 +++++++ 6 files changed, 254 insertions(+), 1 deletion(-) create mode 100755 python/vyos/xml_ref/generate_op_cache.py create mode 100644 python/vyos/xml_ref/op_definition.py diff --git a/.gitignore b/.gitignore index 01333d5b186..c597d9c845d 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,8 @@ data/component-versions.json # vyos-1x XML cache python/vyos/xml_ref/cache.py python/vyos/xml_ref/pkg_cache/*_cache.py +python/vyos/xml_ref/op_cache.py +python/vyos/xml_ref/pkg_cache/*_op_cache.py # autogenerated vyos-configd JSON definition data/configd-include.json diff --git a/Makefile b/Makefile index 685c8f15023..c83380be5a6 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,8 @@ op_mode_definitions: $(op_xml_obj) find $(BUILD_DIR)/op-mode-definitions/ -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-op-templates {} $(CURDIR)/schema/op-mode-definition.rng $(OP_TMPL_DIR) || exit 1 + $(CURDIR)/python/vyos/xml_ref/generate_op_cache.py --xml-dir $(BUILD_DIR)/op-mode-definitions || exit 1 + # XXX: tcpdump, ping, traceroute and mtr must be able to recursivly call themselves as the # options are provided from the scripts themselves ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/ diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 25ee453914a..dec619d3ebf 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -35,7 +35,8 @@ 'vyos_udev_dir' : '/run/udev/vyos', 'isc_dhclient_dir' : '/run/dhclient', 'dhcp6_client_dir' : '/run/dhcp6c', - 'vyos_configdir' : '/opt/vyatta/config' + 'vyos_configdir' : '/opt/vyatta/config', + 'completion_dir' : f'{base_dir}/completion' } config_status = '/tmp/vyos-config-status' diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index 2ba3da4e8ae..91ce394f70d 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -15,6 +15,7 @@ from typing import Optional, Union, TYPE_CHECKING from vyos.xml_ref import definition +from vyos.xml_ref import op_definition if TYPE_CHECKING: from vyos.config import ConfigDict @@ -87,3 +88,25 @@ def from_source(d: dict, path: list) -> bool: def ext_dict_merge(source: dict, destination: Union[dict, 'ConfigDict']): return definition.ext_dict_merge(source, destination) + +def load_op_reference(op_cache=[]): + if op_cache: + return op_cache[0] + + op_xml = op_definition.OpXml() + + try: + from vyos.xml_ref.op_cache import op_reference + except Exception: + raise ImportError('no xml op reference cache !!') + + if not op_reference: + raise ValueError('empty xml op reference cache !!') + + op_xml.define(op_reference) + op_cache.append(op_xml) + + return op_xml + +def get_op_ref_path(path: list) -> list[op_definition.PathData]: + return load_op_reference()._get_op_ref_path(path) diff --git a/python/vyos/xml_ref/generate_op_cache.py b/python/vyos/xml_ref/generate_op_cache.py new file mode 100755 index 00000000000..e93b0797415 --- /dev/null +++ b/python/vyos/xml_ref/generate_op_cache.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# + +import re +import sys +import json +import glob +from argparse import ArgumentParser +from argparse import ArgumentTypeError +from os.path import join +from os.path import abspath +from os.path import dirname +from xml.etree import ElementTree as ET +from xml.etree.ElementTree import Element +from typing import TypeAlias +from typing import Optional + +_here = dirname(__file__) + +sys.path.append(join(_here, '..')) +from defaults import directories + +from op_definition import NodeData +from op_definition import PathData + +xml_op_cache_json = 'xml_op_cache.json' +xml_op_tmp = join('/tmp', xml_op_cache_json) +op_ref_cache = abspath(join(_here, 'op_cache.py')) + +OptElement: TypeAlias = Optional[Element] +DEBUG = False + + +def translate_exec(s: str) -> str: + s = s.replace('${vyos_op_scripts_dir}', directories['op_mode']) + s = s.replace('${vyos_libexec_dir}', directories['base']) + return s + + +def translate_position(s: str, pos: list[str]) -> str: + pos = pos.copy() + pat: re.Pattern = re.compile(r'(?:\")?\${?([0-9]+)}?(?:\")?') + t: str = pat.sub(r'_place_holder_\1_', s) + + # preferred to .format(*list) to avoid collisions with braces + for i, p in enumerate(pos): + t = t.replace(f'_place_holder_{i+1}_', p) + + return t + + +def translate_command(s: str, pos: list[str]) -> str: + s = translate_exec(s) + s = translate_position(s, pos) + return s + + +def translate_op_script(s: str) -> str: + s = s.replace('${vyos_completion_dir}', directories['completion_dir']) + s = s.replace('${vyos_op_scripts_dir}', directories['op_mode']) + return s + + +def insert_node(n: Element, l: list[PathData], path = None) -> None: + # pylint: disable=too-many-locals,too-many-branches + prop: OptElement = n.find('properties') + children: OptElement = n.find('children') + command: OptElement = n.find('command') + # name is not None as required by schema + name: str = n.get('name', 'schema_error') + node_type: str = n.tag + if path is None: + path = [] + + path.append(name) + if node_type == 'tagNode': + path.append(f'{name}-tag_value') + + help_prop: OptElement = None if prop is None else prop.find('help') + help_text = None if help_prop is None else help_prop.text + command_text = None if command is None else command.text + if command_text is not None: + command_text = translate_command(command_text, path) + + comp_help = None + if prop is not None: + che = prop.findall("completionHelp") + for c in che: + lists = c.findall("list") + paths = c.findall("path") + scripts = c.findall("script") + + comp_help = {} + list_l = [] + for i in lists: + list_l.append(i.text) + path_l = [] + for i in paths: + path_str = re.sub(r'\s+', '/', i.text) + path_l.append(path_str) + script_l = [] + for i in scripts: + script_str = translate_op_script(i.text) + script_l.append(script_str) + + comp_help['list'] = list_l + comp_help['fs_path'] = path_l + comp_help['script'] = script_l + + for d in l: + if name in list(d): + break + else: + d = {} + l.append(d) + + inner_l = d.setdefault(name, []) + + inner_d: PathData = {'node_data': NodeData(node_type=node_type, + help_text=help_text, + comp_help=comp_help, + command=command_text, + path=path)} + inner_l.append(inner_d) + + if children is not None: + inner_nodes = children.iterfind("*") + for inner_n in inner_nodes: + inner_path = path[:] + insert_node(inner_n, inner_l, inner_path) + + +def parse_file(file_path, l): + tree = ET.parse(file_path) + root = tree.getroot() + for n in root.iterfind("*"): + insert_node(n, l) + + +def main(): + parser = ArgumentParser(description='generate dict from xml defintions') + parser.add_argument('--xml-dir', type=str, required=True, + help='transcluded xml op-mode-definition file') + + args = vars(parser.parse_args()) + + xml_dir = abspath(args['xml_dir']) + + l = [] + + for fname in glob.glob(f'{xml_dir}/*.xml'): + parse_file(fname, l) + + with open(xml_op_tmp, 'w') as f: + json.dump(l, f, indent=2) + + with open(op_ref_cache, 'w') as f: + f.write(f'op_reference = {str(l)}') + +if __name__ == '__main__': + main() diff --git a/python/vyos/xml_ref/op_definition.py b/python/vyos/xml_ref/op_definition.py new file mode 100644 index 00000000000..914f3a10552 --- /dev/null +++ b/python/vyos/xml_ref/op_definition.py @@ -0,0 +1,49 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +from typing import TypedDict +from typing import TypeAlias +from typing import Optional +from typing import Union + + +class NodeData(TypedDict): + node_type: Optional[str] + help_text: Optional[str] + comp_help: Optional[dict[str, list]] + command: Optional[str] + path: Optional[list[str]] + + +PathData: TypeAlias = dict[str, Union[NodeData|list['PathData']]] + + +class OpXml: + def __init__(self): + self.op_ref = {} + + def define(self, op_ref: list[PathData]) -> None: + self.op_ref = op_ref + + def _get_op_ref_path(self, path: list[str]) -> list[PathData]: + def _get_path_list(path: list[str], l: list[PathData]) -> list[PathData]: + if not path: + return l + for d in l: + if path[0] in list(d): + return _get_path_list(path[1:], d[path[0]]) + return [] + l = self.op_ref + return _get_path_list(path, l) From 5f23b7275564cfaa7c178d320868b5f5e86ae606 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 8 Aug 2024 14:24:00 -0500 Subject: [PATCH 52/64] configverify: T6642: verify_interface_exists requires config_dict arg The function verify_interface_exists requires a reference to the ambient config_dict rather than creating an instance. As access is required to the 'interfaces' path, provide as attribute of class ConfigDict, so as not to confuse path searches of script-specific config_dict instances. --- python/vyos/config.py | 3 +++ python/vyos/configverify.py | 6 ++---- src/conf_mode/firewall.py | 2 +- src/conf_mode/interfaces_ethernet.py | 4 ++-- src/conf_mode/interfaces_wwan.py | 2 +- src/conf_mode/policy_local-route.py | 2 +- src/conf_mode/protocols_igmp-proxy.py | 2 +- src/conf_mode/protocols_isis.py | 2 +- src/conf_mode/protocols_mpls.py | 2 +- src/conf_mode/protocols_ospf.py | 2 +- src/conf_mode/protocols_ospfv3.py | 2 +- src/conf_mode/protocols_pim.py | 2 +- src/conf_mode/protocols_pim6.py | 2 +- src/conf_mode/qos.py | 2 +- src/conf_mode/service_broadcast-relay.py | 2 +- src/conf_mode/service_conntrack-sync.py | 2 +- src/conf_mode/service_dns_dynamic.py | 2 +- src/conf_mode/service_ipoe-server.py | 2 +- src/conf_mode/service_mdns_repeater.py | 2 +- src/conf_mode/service_ndp-proxy.py | 2 +- src/conf_mode/service_ntp.py | 2 +- src/conf_mode/service_pppoe-server.py | 2 +- src/conf_mode/service_salt-minion.py | 2 +- src/conf_mode/system_flow-accounting.py | 2 +- src/conf_mode/system_option.py | 2 +- src/conf_mode/vpn_ipsec.py | 8 ++++---- 26 files changed, 33 insertions(+), 32 deletions(-) diff --git a/python/vyos/config.py b/python/vyos/config.py index b7ee606a928..1fab4676147 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -344,6 +344,9 @@ def get_config_dict(self, path=[], effective=False, key_mangling=None, conf_dict['pki'] = pki_dict + interfaces_root = root_dict.get('interfaces', {}) + setattr(conf_dict, 'interfaces_root', interfaces_root) + # save optional args for a call to get_config_defaults setattr(conf_dict, '_dict_kwargs', kwargs) diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index b49d66c36eb..59b67300dbf 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -237,7 +237,7 @@ def verify_bridge_delete(config): raise ConfigError(f'Interface "{interface}" cannot be deleted as it ' f'is a member of bridge "{bridge_name}"!') -def verify_interface_exists(ifname, state_required=False, warning_only=False): +def verify_interface_exists(config, ifname, state_required=False, warning_only=False): """ Common helper function used by interface implementations to perform recurring validation if an interface actually exists. We first probe @@ -245,14 +245,12 @@ def verify_interface_exists(ifname, state_required=False, warning_only=False): it exists at the OS level. """ from vyos.base import Warning - from vyos.configquery import ConfigTreeQuery from vyos.utils.dict import dict_search_recursive from vyos.utils.network import interface_exists if not state_required: # Check if interface is present in CLI config - config = ConfigTreeQuery() - tmp = config.get_config_dict(['interfaces'], get_first_key=True) + tmp = getattr(config, 'interfaces_root', {}) if bool(list(dict_search_recursive(tmp, ifname))): return True diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index b71ce7124ff..5638a96685b 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -402,7 +402,7 @@ def verify(firewall): raise ConfigError(f'Flowtable "{flowtable}" requires at least one interface') for ifname in flowtable_conf['interface']: - verify_interface_exists(ifname) + verify_interface_exists(firewall, ifname) if dict_search_args(flowtable_conf, 'offload') == 'hardware': interfaces = flowtable_conf['interface'] diff --git a/src/conf_mode/interfaces_ethernet.py b/src/conf_mode/interfaces_ethernet.py index 54d0669cb18..afc48ead8f0 100755 --- a/src/conf_mode/interfaces_ethernet.py +++ b/src/conf_mode/interfaces_ethernet.py @@ -310,7 +310,7 @@ def verify_bond_member(ethernet): :type ethernet: dict """ ifname = ethernet['ifname'] - verify_interface_exists(ifname) + verify_interface_exists(ethernet, ifname) verify_eapol(ethernet) verify_mirror_redirect(ethernet) ethtool = Ethtool(ifname) @@ -327,7 +327,7 @@ def verify_ethernet(ethernet): :type ethernet: dict """ ifname = ethernet['ifname'] - verify_interface_exists(ifname) + verify_interface_exists(ethernet, ifname) verify_mtu(ethernet) verify_mtu_ipv6(ethernet) verify_dhcpv6(ethernet) diff --git a/src/conf_mode/interfaces_wwan.py b/src/conf_mode/interfaces_wwan.py index 2515dc83808..230eb14d68a 100755 --- a/src/conf_mode/interfaces_wwan.py +++ b/src/conf_mode/interfaces_wwan.py @@ -95,7 +95,7 @@ def verify(wwan): if not 'apn' in wwan: raise ConfigError(f'No APN configured for "{ifname}"!') - verify_interface_exists(ifname) + verify_interface_exists(wwan, ifname) verify_authentication(wwan) verify_vrf(wwan) verify_mirror_redirect(wwan) diff --git a/src/conf_mode/policy_local-route.py b/src/conf_mode/policy_local-route.py index f458f4e829c..331fd972dd0 100755 --- a/src/conf_mode/policy_local-route.py +++ b/src/conf_mode/policy_local-route.py @@ -223,7 +223,7 @@ def verify(pbr): if 'inbound_interface' in pbr_route['rule'][rule]: interface = pbr_route['rule'][rule]['inbound_interface'] - verify_interface_exists(interface) + verify_interface_exists(pbr, interface) return None diff --git a/src/conf_mode/protocols_igmp-proxy.py b/src/conf_mode/protocols_igmp-proxy.py index afcef098591..9a07adf0562 100755 --- a/src/conf_mode/protocols_igmp-proxy.py +++ b/src/conf_mode/protocols_igmp-proxy.py @@ -65,7 +65,7 @@ def verify(igmp_proxy): upstream = 0 for interface, config in igmp_proxy['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(igmp_proxy, interface) if dict_search('role', config) == 'upstream': upstream += 1 diff --git a/src/conf_mode/protocols_isis.py b/src/conf_mode/protocols_isis.py index 9cadfd08135..ba2f3cf0d7d 100755 --- a/src/conf_mode/protocols_isis.py +++ b/src/conf_mode/protocols_isis.py @@ -102,7 +102,7 @@ def verify(isis): raise ConfigError('Interface used for routing updates is mandatory!') for interface in isis['interface']: - verify_interface_exists(interface) + verify_interface_exists(isis, interface) # Interface MTU must be >= configured lsp-mtu mtu = Interface(interface).get_mtu() area_mtu = isis['lsp_mtu'] diff --git a/src/conf_mode/protocols_mpls.py b/src/conf_mode/protocols_mpls.py index 177a43444b4..ad164db9f80 100755 --- a/src/conf_mode/protocols_mpls.py +++ b/src/conf_mode/protocols_mpls.py @@ -49,7 +49,7 @@ def verify(mpls): if 'interface' in mpls: for interface in mpls['interface']: - verify_interface_exists(interface) + verify_interface_exists(mpls, interface) # Checks to see if LDP is properly configured if 'ldp' in mpls: diff --git a/src/conf_mode/protocols_ospf.py b/src/conf_mode/protocols_ospf.py index 6fffe7e0d0e..7347c4faac3 100755 --- a/src/conf_mode/protocols_ospf.py +++ b/src/conf_mode/protocols_ospf.py @@ -144,7 +144,7 @@ def verify(ospf): if 'interface' in ospf: for interface, interface_config in ospf['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(ospf, interface) # One can not use dead-interval and hello-multiplier at the same # time. FRR will only activate the last option set via CLI. if {'hello_multiplier', 'dead_interval'} <= set(interface_config): diff --git a/src/conf_mode/protocols_ospfv3.py b/src/conf_mode/protocols_ospfv3.py index 1bb17229324..60c2a9b16b7 100755 --- a/src/conf_mode/protocols_ospfv3.py +++ b/src/conf_mode/protocols_ospfv3.py @@ -127,7 +127,7 @@ def verify(ospfv3): if 'interface' in ospfv3: for interface, interface_config in ospfv3['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(ospfv3, interface) if 'ifmtu' in interface_config: mtu = Interface(interface).get_mtu() if int(interface_config['ifmtu']) > int(mtu): diff --git a/src/conf_mode/protocols_pim.py b/src/conf_mode/protocols_pim.py index d450d11cae9..79294a1f022 100755 --- a/src/conf_mode/protocols_pim.py +++ b/src/conf_mode/protocols_pim.py @@ -97,7 +97,7 @@ def verify(pim): raise ConfigError('PIM require defined interfaces!') for interface, interface_config in pim['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(pim, interface) # Check join group in reserved net if 'igmp' in interface_config and 'join' in interface_config['igmp']: diff --git a/src/conf_mode/protocols_pim6.py b/src/conf_mode/protocols_pim6.py index 2003a10140d..581ffe238a8 100755 --- a/src/conf_mode/protocols_pim6.py +++ b/src/conf_mode/protocols_pim6.py @@ -63,7 +63,7 @@ def verify(pim6): return for interface, interface_config in pim6.get('interface', {}).items(): - verify_interface_exists(interface) + verify_interface_exists(pim6, interface) if 'mld' in interface_config: mld = interface_config['mld'] for group in mld.get('join', {}).keys(): diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 464d7c1920a..7dfad3180e4 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -303,7 +303,7 @@ def apply(qos): return None for interface, interface_config in qos['interface'].items(): - if not verify_interface_exists(interface, state_required=True, warning_only=True): + if not verify_interface_exists(qos, interface, state_required=True, warning_only=True): # When shaper is bound to a dialup (e.g. PPPoE) interface it is # possible that it is yet not availbale when to QoS code runs. # Skip the configuration and inform the user via warning_only=True diff --git a/src/conf_mode/service_broadcast-relay.py b/src/conf_mode/service_broadcast-relay.py index 31c552f5a25..d35954718a9 100755 --- a/src/conf_mode/service_broadcast-relay.py +++ b/src/conf_mode/service_broadcast-relay.py @@ -59,7 +59,7 @@ def verify(relay): raise ConfigError('At least two interfaces are required for UDP broadcast relay "{instance}"') for interface in config.get('interface', []): - verify_interface_exists(interface) + verify_interface_exists(relay, interface) if not is_afi_configured(interface, AF_INET): raise ConfigError(f'Interface "{interface}" has no IPv4 address configured!') diff --git a/src/conf_mode/service_conntrack-sync.py b/src/conf_mode/service_conntrack-sync.py index 4fb2ce27f0f..3a233a1729a 100755 --- a/src/conf_mode/service_conntrack-sync.py +++ b/src/conf_mode/service_conntrack-sync.py @@ -67,7 +67,7 @@ def verify(conntrack): has_peer = False for interface, interface_config in conntrack['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(conntrack, interface) # Interface must not only exist, it must also carry an IP address if len(get_ipv4(interface)) < 1: raise ConfigError(f'Interface {interface} requires an IP address!') diff --git a/src/conf_mode/service_dns_dynamic.py b/src/conf_mode/service_dns_dynamic.py index a551a9891c2..5f530385658 100755 --- a/src/conf_mode/service_dns_dynamic.py +++ b/src/conf_mode/service_dns_dynamic.py @@ -104,7 +104,7 @@ def verify(dyndns): Warning(f'Interface "{config["address"]["interface"]}" does not exist yet and ' f'cannot be used for Dynamic DNS service "{service}" until it is up!') else: - verify_interface_exists(config['address']['interface']) + verify_interface_exists(dyndns, config['address']['interface']) if 'web' in config['address']: # If 'skip' is specified, 'url' is required as well diff --git a/src/conf_mode/service_ipoe-server.py b/src/conf_mode/service_ipoe-server.py index 28b7fb03cd6..16c82e591fd 100755 --- a/src/conf_mode/service_ipoe-server.py +++ b/src/conf_mode/service_ipoe-server.py @@ -66,7 +66,7 @@ def verify(ipoe): raise ConfigError('No IPoE interface configured') for interface, iface_config in ipoe['interface'].items(): - verify_interface_exists(interface, warning_only=True) + verify_interface_exists(ipoe, interface, warning_only=True) if 'client_subnet' in iface_config and 'vlan' in iface_config: raise ConfigError('Option "client-subnet" and "vlan" are mutually exclusive, ' 'use "client-ip-pool" instead!') diff --git a/src/conf_mode/service_mdns_repeater.py b/src/conf_mode/service_mdns_repeater.py index 207da5e03c3..b0ece031cfd 100755 --- a/src/conf_mode/service_mdns_repeater.py +++ b/src/conf_mode/service_mdns_repeater.py @@ -65,7 +65,7 @@ def verify(mdns): # For mdns-repeater to work it is essential that the interfaces has # an IPv4 address assigned for interface in mdns['interface']: - verify_interface_exists(interface) + verify_interface_exists(mdns, interface) if mdns['ip_version'] in ['ipv4', 'both'] and AF_INET not in ifaddresses(interface): raise ConfigError('mDNS repeater requires an IPv4 address to be ' diff --git a/src/conf_mode/service_ndp-proxy.py b/src/conf_mode/service_ndp-proxy.py index aa2374f4cc3..024ad79f22a 100755 --- a/src/conf_mode/service_ndp-proxy.py +++ b/src/conf_mode/service_ndp-proxy.py @@ -50,7 +50,7 @@ def verify(ndpp): if 'interface' in ndpp: for interface, interface_config in ndpp['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(ndpp, interface) if 'rule' in interface_config: for rule, rule_config in interface_config['rule'].items(): diff --git a/src/conf_mode/service_ntp.py b/src/conf_mode/service_ntp.py index f11690ee601..83880fd7266 100755 --- a/src/conf_mode/service_ntp.py +++ b/src/conf_mode/service_ntp.py @@ -64,7 +64,7 @@ def verify(ntp): if 'interface' in ntp: # If ntpd should listen on a given interface, ensure it exists interface = ntp['interface'] - verify_interface_exists(interface) + verify_interface_exists(ntp, interface) # If we run in a VRF, our interface must belong to this VRF, too if 'vrf' in ntp: diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index c95f976d370..566a7b14929 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -122,7 +122,7 @@ def verify(pppoe): # Check is interface exists in the system for interface in pppoe['interface']: - verify_interface_exists(interface, warning_only=True) + verify_interface_exists(pppoe, interface, warning_only=True) return None diff --git a/src/conf_mode/service_salt-minion.py b/src/conf_mode/service_salt-minion.py index a8fce8e018c..edf74b0c0bf 100755 --- a/src/conf_mode/service_salt-minion.py +++ b/src/conf_mode/service_salt-minion.py @@ -70,7 +70,7 @@ def verify(salt): Warning('Do not use sha1 hashing algorithm, upgrade to sha256 or later!') if 'source_interface' in salt: - verify_interface_exists(salt['source_interface']) + verify_interface_exists(salt, salt['source_interface']) return None diff --git a/src/conf_mode/system_flow-accounting.py b/src/conf_mode/system_flow-accounting.py index 2dacd92daf2..a12ee363dce 100755 --- a/src/conf_mode/system_flow-accounting.py +++ b/src/conf_mode/system_flow-accounting.py @@ -183,7 +183,7 @@ def verify(flow_config): # check that all configured interfaces exists in the system for interface in flow_config['interface']: - verify_interface_exists(interface, warning_only=True) + verify_interface_exists(flow_config, interface, warning_only=True) # check sFlow configuration if 'sflow' in flow_config: diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py index 4025104924d..d1647e3a188 100755 --- a/src/conf_mode/system_option.py +++ b/src/conf_mode/system_option.py @@ -68,7 +68,7 @@ def verify(options): if 'http_client' in options: config = options['http_client'] if 'source_interface' in config: - verify_interface_exists(config['source_interface']) + verify_interface_exists(options, config['source_interface']) if {'source_address', 'source_interface'} <= set(config): raise ConfigError('Can not define both HTTP source-interface and source-address') diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index b3e05a814a6..ca0c3657fa1 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -210,9 +210,9 @@ def verify(ipsec): for interface in ipsec['interface']: # exclude check interface for dynamic interfaces if tmp.match(interface): - verify_interface_exists(interface, warning_only=True) + verify_interface_exists(ipsec, interface, warning_only=True) else: - verify_interface_exists(interface) + verify_interface_exists(ipsec, interface) if 'l2tp' in ipsec: if 'esp_group' in ipsec['l2tp']: @@ -273,7 +273,7 @@ def verify(ipsec): if 'dhcp_interface' in ra_conf: dhcp_interface = ra_conf['dhcp_interface'] - verify_interface_exists(dhcp_interface) + verify_interface_exists(ipsec, dhcp_interface) dhcp_base = directories['isc_dhclient_dir'] if not os.path.exists(f'{dhcp_base}/dhclient_{dhcp_interface}.conf'): @@ -502,7 +502,7 @@ def verify(ipsec): if 'dhcp_interface' in peer_conf: dhcp_interface = peer_conf['dhcp_interface'] - verify_interface_exists(dhcp_interface) + verify_interface_exists(ipsec, dhcp_interface) dhcp_base = directories['isc_dhclient_dir'] if not os.path.exists(f'{dhcp_base}/dhclient_{dhcp_interface}.conf'): From a9024f302fd9657a0e6ef274cfc1dedccaf9d1a3 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 30 Jul 2024 08:27:49 -0500 Subject: [PATCH 53/64] configd: T6633: inject missing env vars for configfs utility --- src/services/vyos-configd | 10 ++++++++++ src/shim/vyshim.c | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/services/vyos-configd b/src/services/vyos-configd index 69ee15bf18b..d797e90cfff 100755 --- a/src/services/vyos-configd +++ b/src/services/vyos-configd @@ -182,6 +182,12 @@ def initialization(socket): sudo_user_string = socket.recv().decode("utf-8", "ignore") resp = "sudo_user" socket.send(resp.encode()) + temp_config_dir_string = socket.recv().decode("utf-8", "ignore") + resp = "temp_config_dir" + socket.send(resp.encode()) + changes_only_dir_string = socket.recv().decode("utf-8", "ignore") + resp = "changes_only_dir" + socket.send(resp.encode()) logger.debug(f"config session pid is {pid_string}") logger.debug(f"config session sudo_user is {sudo_user_string}") @@ -198,6 +204,10 @@ def initialization(socket): session_mode = 'a' os.environ['SUDO_USER'] = sudo_user_string + if temp_config_dir_string: + os.environ['VYATTA_TEMP_CONFIG_DIR'] = temp_config_dir_string + if changes_only_dir_string: + os.environ['VYATTA_CHANGES_ONLY_DIR'] = changes_only_dir_string try: configsource = ConfigSourceString(running_config_text=active_string, diff --git a/src/shim/vyshim.c b/src/shim/vyshim.c index 4d836127dd0..a78f62a7b43 100644 --- a/src/shim/vyshim.c +++ b/src/shim/vyshim.c @@ -185,6 +185,20 @@ int initialization(void* Requester) } debug_print("sudo_user is %s\n", sudo_user); + char *temp_config_dir = getenv("VYATTA_TEMP_CONFIG_DIR"); + if (!temp_config_dir) { + char none[] = ""; + temp_config_dir = none; + } + debug_print("temp_config_dir is %s\n", temp_config_dir); + + char *changes_only_dir = getenv("VYATTA_CHANGES_ONLY_DIR"); + if (!changes_only_dir) { + char none[] = ""; + changes_only_dir = none; + } + debug_print("changes_only_dir is %s\n", changes_only_dir); + debug_print("Sending init announcement\n"); char *init_announce = mkjson(MKJSON_OBJ, 1, MKJSON_STRING, "type", "init"); @@ -252,6 +266,16 @@ int initialization(void* Requester) zmq_recv(Requester, buffer, 16, 0); debug_print("Received sudo_user receipt\n"); + debug_print("Sending config session temp_config_dir\n"); + zmq_send(Requester, temp_config_dir, strlen(temp_config_dir), 0); + zmq_recv(Requester, buffer, 16, 0); + debug_print("Received temp_config_dir receipt\n"); + + debug_print("Sending config session changes_only_dir\n"); + zmq_send(Requester, changes_only_dir, strlen(changes_only_dir), 0); + zmq_recv(Requester, buffer, 16, 0); + debug_print("Received changes_only_dir receipt\n"); + return 0; } From a2b6098e6f9c1915a61a9aebc8f9852bd897387c Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Sun, 11 Aug 2024 15:18:38 -0700 Subject: [PATCH 54/64] T6648: dhcpv6-server: align stateless DHCPv6 options with stateful --- .../include/dhcp/option-v6.xml.i | 12 +++++++ .../version/dhcpv6-server-version.xml.i | 2 +- .../service_dhcpv6-server.xml.in | 22 +------------ python/vyos/template.py | 4 +-- src/migration-scripts/dhcpv6-server/5-to-6 | 31 +++++++++++++++++++ 5 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 src/migration-scripts/dhcpv6-server/5-to-6 diff --git a/interface-definitions/include/dhcp/option-v6.xml.i b/interface-definitions/include/dhcp/option-v6.xml.i index 1df0c3934db..e1897f52d95 100644 --- a/interface-definitions/include/dhcp/option-v6.xml.i +++ b/interface-definitions/include/dhcp/option-v6.xml.i @@ -78,6 +78,18 @@ + + + Time (in seconds) that stateless clients should wait between refreshing the information they were given + + u32:1-4294967295 + DHCPv6 information refresh time + + + + + + Vendor Specific Options diff --git a/interface-definitions/include/version/dhcpv6-server-version.xml.i b/interface-definitions/include/version/dhcpv6-server-version.xml.i index 1f30368a322..8b72a9c7202 100644 --- a/interface-definitions/include/version/dhcpv6-server-version.xml.i +++ b/interface-definitions/include/version/dhcpv6-server-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/service_dhcpv6-server.xml.in b/interface-definitions/service_dhcpv6-server.xml.in index daca7b43fb1..cf14388e8ac 100644 --- a/interface-definitions/service_dhcpv6-server.xml.in +++ b/interface-definitions/service_dhcpv6-server.xml.in @@ -63,27 +63,7 @@ - - - Common options to distribute to all clients, including stateless clients - - - - - Time (in seconds) that stateless clients should wait between refreshing the information they were given - - u32:1-4294967295 - DHCPv6 information refresh time - - - - - - - #include - #include - - + #include IPv6 DHCP subnet for this shared network diff --git a/python/vyos/template.py b/python/vyos/template.py index 3507e0940f7..aa99bed5a6c 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -922,8 +922,8 @@ def kea6_shared_network_json(shared_networks): 'subnet6': [] } - if 'common_options' in config: - network['option-data'] = kea6_parse_options(config['common_options']) + if 'option' in config: + network['option-data'] = kea6_parse_options(config['option']) if 'interface' in config: network['interface'] = config['interface'] diff --git a/src/migration-scripts/dhcpv6-server/5-to-6 b/src/migration-scripts/dhcpv6-server/5-to-6 new file mode 100644 index 00000000000..cad0a353881 --- /dev/null +++ b/src/migration-scripts/dhcpv6-server/5-to-6 @@ -0,0 +1,31 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +# T6648: Rename "common-options" to "option" at shared-network level + +from vyos.configtree import ConfigTree + +base = ['service', 'dhcpv6-server', 'shared-network-name'] + +def migrate(config: ConfigTree) -> None: + if not config.exists(base): + # Nothing to do + return + + for network in config.list_nodes(base): + if not config.exists(base + [network, 'common-options']): + continue + + config.rename(base + [network, 'common-options'], 'option') From ba87bed16080bc799f83914e48693dac880386e2 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Thu, 8 Aug 2024 13:09:43 +0300 Subject: [PATCH 55/64] suricata: T6624: Fix for service suricata address-groups cannot be used in each other --- src/conf_mode/service_suricata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conf_mode/service_suricata.py b/src/conf_mode/service_suricata.py index 69b369e0b70..1ce1701458f 100755 --- a/src/conf_mode/service_suricata.py +++ b/src/conf_mode/service_suricata.py @@ -59,7 +59,7 @@ def visit(n, v): temporary_marks.add(n) for m in v.get('group', []): - m = m.lstrip('!') + m = m.lstrip('!').replace('-', '_') if m not in source: raise ConfigError(f'Undefined referenced group "{m}"') visit(m, source[m]) From 69ab44309d56d73d92c2f8a7b0b4ca3016e61ff6 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Wed, 14 Aug 2024 13:20:46 +0300 Subject: [PATCH 56/64] op_mode: T6651: Add a top level op mode word "execute" --- op-mode-definitions/execute.xml.in | 8 ++++++++ python/vyos/opmode.py | 2 +- src/opt/vyatta/etc/shell/level/users/allowed-op | 1 + src/opt/vyatta/etc/shell/level/users/allowed-op.in | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 op-mode-definitions/execute.xml.in diff --git a/op-mode-definitions/execute.xml.in b/op-mode-definitions/execute.xml.in new file mode 100644 index 00000000000..66069c92757 --- /dev/null +++ b/op-mode-definitions/execute.xml.in @@ -0,0 +1,8 @@ + + + + + Initiate an operation + + + \ No newline at end of file diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index a6c64adfb62..066c8058f23 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -89,7 +89,7 @@ class InternalError(Error): def _is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart|add|update|delete|generate|set|renew|release)", name): + if re.match(r"^(show|clear|reset|restart|add|update|delete|generate|set|renew|release|execute)", name): return True else: return False diff --git a/src/opt/vyatta/etc/shell/level/users/allowed-op b/src/opt/vyatta/etc/shell/level/users/allowed-op index 74c45af37e2..381fd26e564 100644 --- a/src/opt/vyatta/etc/shell/level/users/allowed-op +++ b/src/opt/vyatta/etc/shell/level/users/allowed-op @@ -6,6 +6,7 @@ clear connect delete disconnect +execute exit force monitor diff --git a/src/opt/vyatta/etc/shell/level/users/allowed-op.in b/src/opt/vyatta/etc/shell/level/users/allowed-op.in index 1976904e4d1..9752f99a218 100644 --- a/src/opt/vyatta/etc/shell/level/users/allowed-op.in +++ b/src/opt/vyatta/etc/shell/level/users/allowed-op.in @@ -2,6 +2,7 @@ clear connect delete disconnect +execute exit force monitor From 2d953bedd0e416ead924f77ec612c997f950535a Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 14 Aug 2024 12:12:56 +0000 Subject: [PATCH 57/64] T6646: conntrack: in ignore rules, if protocols=all, do not append it to the rule --- python/vyos/template.py | 3 ++- smoketest/scripts/cli/test_system_conntrack.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 python/vyos/template.py diff --git a/python/vyos/template.py b/python/vyos/template.py old mode 100644 new mode 100755 index aa99bed5a6c..be9f781a614 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -694,7 +694,8 @@ def conntrack_rule(rule_conf, rule_id, action, ipv6=False): else: for protocol, protocol_config in rule_conf['protocol'].items(): proto = protocol - output.append(f'meta l4proto {proto}') + if proto != 'all': + output.append(f'meta l4proto {proto}') tcp_flags = dict_search_args(rule_conf, 'tcp', 'flags') if tcp_flags and action != 'timeout': diff --git a/smoketest/scripts/cli/test_system_conntrack.py b/smoketest/scripts/cli/test_system_conntrack.py index c07fdce778b..72deb752558 100755 --- a/smoketest/scripts/cli/test_system_conntrack.py +++ b/smoketest/scripts/cli/test_system_conntrack.py @@ -209,6 +209,7 @@ def test_conntrack_ignore(self): self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '2', 'source', 'address', '192.0.2.1']) self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '2', 'destination', 'group', 'address-group', address_group]) + self.cli_set(base_path + ['ignore', 'ipv4', 'rule', '2', 'protocol', 'all']) self.cli_set(base_path + ['ignore', 'ipv6', 'rule', '11', 'source', 'address', 'fe80::1']) self.cli_set(base_path + ['ignore', 'ipv6', 'rule', '11', 'destination', 'address', 'fe80::2']) From 747363e3ecd303e515cccdac381f321683613e20 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 14 Aug 2024 14:53:14 +0000 Subject: [PATCH 58/64] T6636: firewall: fix firewall template in order to write logs for default-action in order to match same structure as in rules. This way op-mode command for showing firewall log prints logs for default-actions too --- data/templates/firewall/nftables.j2 | 6 +++--- smoketest/scripts/cli/test_firewall.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) mode change 100644 => 100755 data/templates/firewall/nftables.j2 diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 old mode 100644 new mode 100755 index 82dcefac00a..155b7f4d0c0 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -135,7 +135,7 @@ table ip vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule(name_text, 'ipv4') }} + {{ conf | nft_default_rule('NAM-' + name_text, 'ipv4') }} } {% endfor %} {% endif %} @@ -287,7 +287,7 @@ table ip6 vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule(name_text, 'ipv6') }} + {{ conf | nft_default_rule('NAM-' + name_text, 'ipv6') }} } {% endfor %} {% endif %} @@ -416,7 +416,7 @@ table bridge vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule(name_text, 'bri') }} + {{ conf | nft_default_rule('NAM-' + name_text, 'bri') }} } {% endfor %} {% endif %} diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 8aeeff1496b..b8031eed069 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -280,7 +280,7 @@ def test_ipv4_basic_rules(self): ['chain NAME_smoketest'], ['saddr 172.16.20.10', 'daddr 172.16.10.10', 'log prefix "[ipv4-NAM-smoketest-1-A]" log level debug', 'ip ttl 15', 'accept'], ['tcp flags syn / syn,ack', 'tcp dport 8888', 'log prefix "[ipv4-NAM-smoketest-2-R]" log level err', 'ip ttl > 102', 'reject'], - ['log prefix "[ipv4-smoketest-default-D]"','smoketest default-action', 'drop'] + ['log prefix "[ipv4-NAM-smoketest-default-D]"','smoketest default-action', 'drop'] ] self.verify_nftables(nftables_search, 'ip vyos_filter') @@ -341,7 +341,7 @@ def test_ipv4_advanced(self): [f'chain NAME_{name}'], ['ip length { 64, 512, 1024 }', 'ip dscp { 0x11, 0x34 }', f'log prefix "[ipv4-NAM-{name}-6-A]" log group 66 snaplen 6666 queue-threshold 32000', 'accept'], ['ip length 1-30000', 'ip length != 60000-65535', 'ip dscp 0x03-0x0b', 'ip dscp != 0x15-0x19', 'accept'], - [f'log prefix "[ipv4-{name}-default-D]"', 'drop'] + [f'log prefix "[ipv4-NAM-{name}-default-D]"', 'drop'] ] self.verify_nftables(nftables_search, 'ip vyos_filter') @@ -511,7 +511,7 @@ def test_ipv6_basic_rules(self): ['PRE-raw default-action accept', 'accept'], [f'chain NAME6_{name}'], ['saddr 2002::1-2002::10', 'daddr 2002::1:1', 'log prefix "[ipv6-NAM-v6-smoketest-1-A]" log level crit', 'accept'], - [f'"{name} default-action drop"', f'log prefix "[ipv6-{name}-default-D]"', 'drop'], + [f'"NAM-{name} default-action drop"', f'log prefix "[ipv6-NAM-{name}-default-D]"', 'drop'], ['jump VYOS_STATE_POLICY6'], ['chain VYOS_STATE_POLICY6'], ['ct state established', 'accept'], @@ -522,9 +522,7 @@ def test_ipv6_basic_rules(self): self.verify_nftables(nftables_search, 'ip6 vyos_filter') def test_ipv6_advanced(self): - name = 'v6-smoketest-adv' - name2 = 'v6-smoketest-adv2' - interface = 'eth0' + name = 'v6-smoke-adv' self.cli_set(['firewall', 'ipv6', 'name', name, 'default-action', 'drop']) self.cli_set(['firewall', 'ipv6', 'name', name, 'default-log']) @@ -559,7 +557,7 @@ def test_ipv6_advanced(self): ['ip6 saddr 2001:db8::/64', 'meta mark != 0x000019ff-0x00001e56', f'jump NAME6_{name}'], [f'chain NAME6_{name}'], ['ip6 length { 65, 513, 1025 }', 'ip6 dscp { af21, 0x35 }', 'accept'], - [f'log prefix "[ipv6-{name}-default-D]"', 'drop'] + [f'log prefix "[ipv6-NAM-{name}-default-D]"', 'drop'] ] self.verify_nftables(nftables_search, 'ip6 vyos_filter') @@ -686,7 +684,7 @@ def test_ipv4_state_and_status_rules(self): ['ct state new', 'ct status dnat', 'accept'], ['ct state { established, new }', 'ct status snat', 'accept'], ['ct state related', 'ct helper { "ftp", "pptp" }', 'accept'], - ['drop', f'comment "{name} default-action drop"'] + ['drop', f'comment "NAM-{name} default-action drop"'] ] self.verify_nftables(nftables_search, 'ip vyos_filter') From 663e468de2b431f771534b4e3a2d00a5924b98fe Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Thu, 15 Aug 2024 13:20:31 +0300 Subject: [PATCH 59/64] T6649: Accel-ppp separate vlan-mon from listen interfaces --- data/templates/accel-ppp/ipoe.config.j2 | 2 +- data/templates/accel-ppp/pppoe.config.j2 | 2 ++ .../include/accel-ppp/vlan-mon.xml.i | 8 +++++ .../include/version/ipoe-server-version.xml.i | 2 +- .../version/pppoe-server-version.xml.i | 2 +- .../service_ipoe-server.xml.in | 1 + .../service_pppoe-server.xml.in | 1 + .../scripts/cli/test_service_ipoe-server.py | 32 +++++++++++++++++++ .../scripts/cli/test_service_pppoe-server.py | 7 ++++ src/conf_mode/service_ipoe-server.py | 2 ++ src/conf_mode/service_pppoe-server.py | 5 ++- src/migration-scripts/ipoe-server/3-to-4 | 30 +++++++++++++++++ src/migration-scripts/pppoe-server/10-to-11 | 30 +++++++++++++++++ 13 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 interface-definitions/include/accel-ppp/vlan-mon.xml.i create mode 100644 src/migration-scripts/ipoe-server/3-to-4 create mode 100644 src/migration-scripts/pppoe-server/10-to-11 diff --git a/data/templates/accel-ppp/ipoe.config.j2 b/data/templates/accel-ppp/ipoe.config.j2 index 9729b295e62..81f63c53b05 100644 --- a/data/templates/accel-ppp/ipoe.config.j2 +++ b/data/templates/accel-ppp/ipoe.config.j2 @@ -56,7 +56,7 @@ verbose=1 {% set relay = ',' ~ 'relay=' ~ iface_config.external_dhcp.dhcp_relay if iface_config.external_dhcp.dhcp_relay is vyos_defined else '' %} {% set giaddr = ',' ~ 'giaddr=' ~ iface_config.external_dhcp.giaddr if iface_config.external_dhcp.giaddr is vyos_defined else '' %} {{ tmp }},{{ shared }}mode={{ iface_config.mode | upper }},ifcfg=1,{{ range }}start=dhcpv4,ipv6=1{{ relay }}{{ giaddr }} -{% if iface_config.vlan is vyos_defined %} +{% if iface_config.vlan_mon is vyos_defined %} vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }} {% endif %} {% endfor %} diff --git a/data/templates/accel-ppp/pppoe.config.j2 b/data/templates/accel-ppp/pppoe.config.j2 index 73ffe09631f..beab46936e6 100644 --- a/data/templates/accel-ppp/pppoe.config.j2 +++ b/data/templates/accel-ppp/pppoe.config.j2 @@ -61,7 +61,9 @@ interface={{ iface }} {% for vlan in iface_config.vlan %} interface=re:^{{ iface }}\.{{ vlan | range_to_regex }}$ {% endfor %} +{% if iface_config.vlan_mon is vyos_defined %} vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }} +{% endif %} {% endif %} {% endfor %} {% endif %} diff --git a/interface-definitions/include/accel-ppp/vlan-mon.xml.i b/interface-definitions/include/accel-ppp/vlan-mon.xml.i new file mode 100644 index 00000000000..d5bacb0d157 --- /dev/null +++ b/interface-definitions/include/accel-ppp/vlan-mon.xml.i @@ -0,0 +1,8 @@ + + + + Automatically create VLAN interfaces + + + + diff --git a/interface-definitions/include/version/ipoe-server-version.xml.i b/interface-definitions/include/version/ipoe-server-version.xml.i index 6594333822b..b7718fc5e74 100644 --- a/interface-definitions/include/version/ipoe-server-version.xml.i +++ b/interface-definitions/include/version/ipoe-server-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/include/version/pppoe-server-version.xml.i b/interface-definitions/include/version/pppoe-server-version.xml.i index 61de1277a57..2e020faa366 100644 --- a/interface-definitions/include/version/pppoe-server-version.xml.i +++ b/interface-definitions/include/version/pppoe-server-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/service_ipoe-server.xml.in b/interface-definitions/service_ipoe-server.xml.in index c7542f0d05c..25bc43cc650 100644 --- a/interface-definitions/service_ipoe-server.xml.in +++ b/interface-definitions/service_ipoe-server.xml.in @@ -175,6 +175,7 @@ #include + #include #include diff --git a/interface-definitions/service_pppoe-server.xml.in b/interface-definitions/service_pppoe-server.xml.in index 7cb1ec06e77..93ec7ade90d 100644 --- a/interface-definitions/service_pppoe-server.xml.in +++ b/interface-definitions/service_pppoe-server.xml.in @@ -64,6 +64,7 @@ #include + #include diff --git a/smoketest/scripts/cli/test_service_ipoe-server.py b/smoketest/scripts/cli/test_service_ipoe-server.py index 5f1cf9ad173..be03179bfc5 100755 --- a/smoketest/scripts/cli/test_service_ipoe-server.py +++ b/smoketest/scripts/cli/test_service_ipoe-server.py @@ -21,6 +21,7 @@ from base_accel_ppp_test import BasicAccelPPPTest from vyos.configsession import ConfigSessionError from vyos.utils.process import cmd +from vyos.template import range_to_regex from configparser import ConfigParser from configparser import RawConfigParser @@ -228,6 +229,37 @@ def test_accel_ipv6_pool(self): delegate={delegate_2_prefix},{delegate_mask},name={pool_name}""" self.assertIn(pool_config, config) + def test_ipoe_server_vlan(self): + vlans = ['100', '200', '300-310'] + + # Test configuration of local authentication for PPPoE server + self.basic_config() + # cannot use "client-subnet" option with "vlan" option + # have to delete it + self.delete(['interface', interface, 'client-subnet']) + self.cli_commit() + + self.set(['interface', interface, 'vlan-mon']) + + # cannot use option "vlan-mon" if no "vlan" set + with self.assertRaises(ConfigSessionError): + self.cli_commit() + + for vlan in vlans: + self.set(['interface', interface, 'vlan', vlan]) + + # commit changes + self.cli_commit() + + # Validate configuration values + conf = ConfigParser(allow_no_value=True, delimiters='=', strict=False) + conf.read(self._config_file) + tmp = range_to_regex(vlans) + self.assertIn(f're:^{interface}\.{tmp}$', conf['ipoe']['interface']) + + tmp = ','.join(vlans) + self.assertIn(f'{interface},{tmp}', conf['ipoe']['vlan-mon']) + @unittest.skip("PPP is not a part of IPoE") def test_accel_ppp_options(self): pass diff --git a/smoketest/scripts/cli/test_service_pppoe-server.py b/smoketest/scripts/cli/test_service_pppoe-server.py index 34e45a81a19..8add5ee6cbd 100755 --- a/smoketest/scripts/cli/test_service_pppoe-server.py +++ b/smoketest/scripts/cli/test_service_pppoe-server.py @@ -21,6 +21,7 @@ from configparser import ConfigParser from vyos.utils.file import read_file from vyos.template import range_to_regex +from vyos.configsession import ConfigSessionError local_if = ['interfaces', 'dummy', 'dum667'] ac_name = 'ACN' @@ -133,6 +134,12 @@ def test_pppoe_server_vlan(self): # Test configuration of local authentication for PPPoE server self.basic_config() + self.set(['interface', interface, 'vlan-mon']) + + # cannot use option "vlan-mon" if no "vlan" set + with self.assertRaises(ConfigSessionError): + self.cli_commit() + for vlan in vlans: self.set(['interface', interface, 'vlan', vlan]) diff --git a/src/conf_mode/service_ipoe-server.py b/src/conf_mode/service_ipoe-server.py index 16c82e591fd..c7e3ef0336c 100755 --- a/src/conf_mode/service_ipoe-server.py +++ b/src/conf_mode/service_ipoe-server.py @@ -70,6 +70,8 @@ def verify(ipoe): if 'client_subnet' in iface_config and 'vlan' in iface_config: raise ConfigError('Option "client-subnet" and "vlan" are mutually exclusive, ' 'use "client-ip-pool" instead!') + if 'vlan_mon' in iface_config and not 'vlan' in iface_config: + raise ConfigError('Option "vlan-mon" requires "vlan" to be set!') verify_accel_ppp_authentication(ipoe, local_users=False) verify_accel_ppp_ip_pool(ipoe) diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index 566a7b14929..ac697c50920 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -121,9 +121,12 @@ def verify(pppoe): raise ConfigError('At least one listen interface must be defined!') # Check is interface exists in the system - for interface in pppoe['interface']: + for interface, interface_config in pppoe['interface'].items(): verify_interface_exists(pppoe, interface, warning_only=True) + if 'vlan_mon' in interface_config and not 'vlan' in interface_config: + raise ConfigError('Option "vlan-mon" requires "vlan" to be set!') + return None diff --git a/src/migration-scripts/ipoe-server/3-to-4 b/src/migration-scripts/ipoe-server/3-to-4 new file mode 100644 index 00000000000..3bad9756d4f --- /dev/null +++ b/src/migration-scripts/ipoe-server/3-to-4 @@ -0,0 +1,30 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +# Add the "vlan-mon" option to the configuration to prevent it +# from disappearing from the configuration file + +from vyos.configtree import ConfigTree + +base = ['service', 'ipoe-server'] + +def migrate(config: ConfigTree) -> None: + if not config.exists(base): + return + + for interface in config.list_nodes(base + ['interface']): + base_path = base + ['interface', interface] + if config.exists(base_path + ['vlan']): + config.set(base_path + ['vlan-mon']) diff --git a/src/migration-scripts/pppoe-server/10-to-11 b/src/migration-scripts/pppoe-server/10-to-11 new file mode 100644 index 00000000000..6bc138b5c25 --- /dev/null +++ b/src/migration-scripts/pppoe-server/10-to-11 @@ -0,0 +1,30 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +# Add the "vlan-mon" option to the configuration to prevent it +# from disappearing from the configuration file + +from vyos.configtree import ConfigTree + +base = ['service', 'pppoe-server'] + +def migrate(config: ConfigTree) -> None: + if not config.exists(base): + return + + for interface in config.list_nodes(base + ['interface']): + base_path = base + ['interface', interface] + if config.exists(base_path + ['vlan']): + config.set(base_path + ['vlan-mon']) From b3ae35987a860a5d2cf64dfbc156a7ee7cc799a2 Mon Sep 17 00:00:00 2001 From: nicolas Date: Thu, 15 Aug 2024 14:37:59 -0300 Subject: [PATCH 60/64] T5794: change firewall priority in oder to be loaded after all interfaces. --- interface-definitions/firewall.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 interface-definitions/firewall.xml.in diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in old mode 100644 new mode 100755 index 816dd1855fc..07c88f799e6 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -2,7 +2,7 @@ - 319 + 489 Firewall From 58125b64c6678ea581998c9f83a19fae0cdbda12 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 15 Aug 2024 13:29:21 -0500 Subject: [PATCH 61/64] utils: T6658: fix write_file check in case of empty directory path --- python/vyos/utils/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/vyos/utils/file.py b/python/vyos/utils/file.py index c566f0334af..eaebb57a3ed 100644 --- a/python/vyos/utils/file.py +++ b/python/vyos/utils/file.py @@ -51,7 +51,7 @@ def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=N If directory of file is not present, it is auto-created. """ dirname = os.path.dirname(fname) - if not os.path.isdir(dirname): + if dirname and not os.path.isdir(dirname): os.makedirs(dirname, mode=0o755, exist_ok=False) chown(dirname, user, group) From 003209eeab231675e82abb8cf6eab7ca0384bc3f Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Fri, 16 Aug 2024 02:18:03 -0700 Subject: [PATCH 62/64] T6659: suricata: use unique cluster_id per interface (#3992) --- data/templates/ids/suricata.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/ids/suricata.j2 b/data/templates/ids/suricata.j2 index 585db93ebf3..d76994c471f 100644 --- a/data/templates/ids/suricata.j2 +++ b/data/templates/ids/suricata.j2 @@ -79,7 +79,7 @@ af-packet: {% for interface in suricata.interface %} - interface: {{ interface }} # Default clusterid. AF_PACKET will load balance packets based on flow. - cluster-id: 99 + cluster-id: {{ 100 - loop.index }} # Default AF_PACKET cluster type. AF_PACKET can load balance per flow or per hash. # This is only supported for Linux kernel > 3.1 # possible value are: From 94ace019c8d37f364abd46723fbbabcd8473cade Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 18 Aug 2024 08:02:51 +0200 Subject: [PATCH 63/64] xml: T6650: fix unused ArgumentTypeError imported from argparse --- python/vyos/xml_ref/generate_op_cache.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/vyos/xml_ref/generate_op_cache.py b/python/vyos/xml_ref/generate_op_cache.py index e93b0797415..cd2ac890eb3 100755 --- a/python/vyos/xml_ref/generate_op_cache.py +++ b/python/vyos/xml_ref/generate_op_cache.py @@ -13,15 +13,13 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# -# import re import sys import json import glob + from argparse import ArgumentParser -from argparse import ArgumentTypeError from os.path import join from os.path import abspath from os.path import dirname From 71d6d0fe31db13f4ddf5c75209b9bba88a1e0a32 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Sat, 17 Aug 2024 13:26:51 +0300 Subject: [PATCH 64/64] op_mode: T3961: Generate PKI expect 2 character country code --- src/op_mode/pki.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/op_mode/pki.py b/src/op_mode/pki.py index 84b0800235d..ea7e93931df 100755 --- a/src/op_mode/pki.py +++ b/src/op_mode/pki.py @@ -316,7 +316,13 @@ def generate_certificate_request(private_key=None, key_type=None, return_request default_values = get_default_values() subject = {} - subject['country'] = ask_input('Enter country code:', default=default_values['country']) + while True: + country = ask_input('Enter country code:', default=default_values['country']) + if len(country) != 2: + print("Country name must be a 2 character country code") + continue + subject['country'] = country + break subject['state'] = ask_input('Enter state:', default=default_values['state']) subject['locality'] = ask_input('Enter locality:', default=default_values['locality']) subject['organization'] = ask_input('Enter organization name:', default=default_values['organization'])