Merge branch 'vyos:current' into circinus_shoretel

vyos · Aug 19, 2024 · 512f0cc · 512f0cc
2 parents 3c48e28 + 26ebd3a
commit 512f0cc
Show file tree

Hide file tree

Showing 148 changed files with 3,803 additions and 535 deletions.
diff --git a/.github/workflows/chceck-pr-message.yml → .github/workflows/check-pr-message.yml b/.github/workflows/chceck-pr-message.yml → .github/workflows/check-pr-message.yml
diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml
@@ -3,11 +3,6 @@ on:
   pull_request_target:
     branches:
       - current
-    paths:
-      - '**'
-      - '!.github/**'
-      - '!**/*.md'
-  workflow_dispatch:
 
 permissions:
   pull-requests: write

diff --git a/.github/workflows/package-smoketest.yml b/.github/workflows/package-smoketest.yml
@@ -1,7 +1,7 @@
 name: VyOS ISO integration Test
 
 on:
-  pull_request_target:
+  pull_request:
     branches:
       - current
     paths:
@@ -37,6 +37,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: packages/vyos-1x
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
       - name: Build vyos-1x package
         run: |
           cd packages/vyos-1x; dpkg-buildpackage -uc -us -tc -b

diff --git a/.github/workflows/trigger-pr.yml b/.github/workflows/trigger-pr.yml
@@ -5,13 +5,13 @@ on:
       types:
         - closed
       branches:
-        - current
-        
+        - circinus
+
 jobs:
   trigger-PR:
     uses: vyos/.github/.github/workflows/trigger-pr.yml@current
     with:
-      source_branch: 'current'
+      source_branch: 'circinus'
       target_branch: 'circinus'
     secrets:
       REMOTE_REPO: ${{ secrets.REMOTE_REPO }}

diff --git a/.gitignore b/.gitignore
@@ -145,6 +145,8 @@ data/component-versions.json
 # vyos-1x XML cache
 python/vyos/xml_ref/cache.py
 python/vyos/xml_ref/pkg_cache/*_cache.py
+python/vyos/xml_ref/op_cache.py
+python/vyos/xml_ref/pkg_cache/*_op_cache.py
 # autogenerated vyos-configd JSON definition
 data/configd-include.json
 

diff --git a/Makefile b/Makefile
@@ -55,6 +55,8 @@ op_mode_definitions: $(op_xml_obj)
 
 	find $(BUILD_DIR)/op-mode-definitions/ -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-op-templates {} $(CURDIR)/schema/op-mode-definition.rng $(OP_TMPL_DIR) || exit 1
 
+	$(CURDIR)/python/vyos/xml_ref/generate_op_cache.py --xml-dir $(BUILD_DIR)/op-mode-definitions || exit 1
+
 	# XXX: tcpdump, ping, traceroute and mtr must be able to recursivly call themselves as the
 	# options are provided from the scripts themselves
 	ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/

diff --git a/README.md b/README.md
@@ -71,3 +71,9 @@ Runtime tests are executed by the CI system on a running VyOS instance inside
 QEMU. The testcases can be found inside the smoketest subdirectory which will
 be placed into the vyos-1x-smoketest package.
 
+
+### Thanks to all the people who already contributed!
+
+<a href="https://github.com/vyos/vyos-1x/graphs/contributors">
+  <img src="https://contributors-img.web.app/image?repo=vyos/vyos-1x" />
+</a>
diff --git a/data/config-mode-dependencies/vyos-1x.json b/data/config-mode-dependencies/vyos-1x.json
@@ -64,8 +64,14 @@
     "system_wireless": {
         "wireless": ["interfaces_wireless"]
     },
+    "system_ip": {
+        "sysctl": ["system_sysctl"]
+    },
+    "system_ipv6": {
+        "sysctl": ["system_sysctl"]
+    },
     "system_option": {
-        "ip": ["system_ip"],
-        "ipv6": ["system_ipv6"]
+        "ip_ipv6": ["system_ip", "system_ipv6"],
+        "sysctl": ["system_sysctl"]
     }
 }
diff --git a/data/templates/accel-ppp/ipoe.config.j2 b/data/templates/accel-ppp/ipoe.config.j2
@@ -56,7 +56,7 @@ verbose=1
 {%         set relay = ',' ~ 'relay=' ~ iface_config.external_dhcp.dhcp_relay  if iface_config.external_dhcp.dhcp_relay is vyos_defined else '' %}
 {%         set giaddr = ',' ~ 'giaddr=' ~ iface_config.external_dhcp.giaddr if iface_config.external_dhcp.giaddr is vyos_defined else '' %}
 {{ tmp }},{{ shared }}mode={{ iface_config.mode | upper }},ifcfg=1,{{ range }}start=dhcpv4,ipv6=1{{ relay }}{{ giaddr }}
-{%         if iface_config.vlan is vyos_defined %}
+{%         if iface_config.vlan_mon is vyos_defined %}
 vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }}
 {%         endif %}
 {%     endfor %}

diff --git a/data/templates/accel-ppp/pppoe.config.j2 b/data/templates/accel-ppp/pppoe.config.j2
@@ -61,7 +61,9 @@ interface={{ iface }}
 {%             for vlan in iface_config.vlan %}
 interface=re:^{{ iface }}\.{{ vlan | range_to_regex }}$
 {%             endfor %}
+{%             if iface_config.vlan_mon is vyos_defined %}
 vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }}
+{%             endif %}
 {%         endif %}
 {%     endfor %}
 {% endif %}

diff --git a/data/templates/conntrack/sysctl.conf.j2 b/data/templates/conntrack/sysctl.conf.j2
@@ -6,4 +6,5 @@ net.netfilter.nf_conntrack_max = {{ table_size }}
 net.ipv4.tcp_max_syn_backlog = {{ tcp.half_open_connections }}
 net.netfilter.nf_conntrack_tcp_loose = {{ '1' if tcp.loose is vyos_defined('enable') else '0' }}
 net.netfilter.nf_conntrack_tcp_max_retrans = {{ tcp.max_retrans }}
-net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
+net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
+net.netfilter.nf_conntrack_timestamp = {{ '1' if log.timestamp is vyos_defined else '0' }}
diff --git a/data/templates/firewall/nftables-bridge.j2 b/data/templates/firewall/nftables-bridge.j2
@@ -1,9 +1,13 @@
+{% import 'firewall/nftables-defines.j2' as group_tmpl %}
 {% macro bridge(bridge) %}
 {% set ns = namespace(sets=[]) %}
 {% if bridge.forward is vyos_defined %}
 {%     for prior, conf in bridge.forward.items() %}
     chain VYOS_FORWARD_{{ prior }} {
         type filter hook forward priority {{ prior }}; policy accept;
+{%         if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%         endif %}
 {%         if conf.rule is vyos_defined %}
 {%             for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
     {{ rule_conf | nft_rule('FWD', prior, rule_id, 'bri') }}
@@ -17,6 +21,46 @@
 {%     endfor %}
 {% endif %}
 
+{% if bridge.input is vyos_defined %}
+{%     for prior, conf in bridge.input.items() %}
+    chain VYOS_INPUT_{{ prior }} {
+        type filter hook input priority {{ prior }}; policy accept;
+{%         if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%         endif %}
+{%         if conf.rule is vyos_defined %}
+{%             for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+    {{ rule_conf | nft_rule('INP', prior, rule_id, 'bri') }}
+{%                 if rule_conf.recent is vyos_defined %}
+{%                     set ns.sets = ns.sets + ['INP_' + prior + '_' + rule_id] %}
+{%                 endif %}
+{%             endfor %}
+{%         endif %}
+    {{ conf | nft_default_rule('INP-filter', 'bri') }}
+    }
+{%     endfor %}
+{% endif %}
+
+{% if bridge.output is vyos_defined %}
+{%     for prior, conf in bridge.output.items() %}
+    chain VYOS_OUTUT_{{ prior }} {
+        type filter hook output priority {{ prior }}; policy accept;
+{%         if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%         endif %}
+{%         if conf.rule is vyos_defined %}
+{%             for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+    {{ rule_conf | nft_rule('OUT', prior, rule_id, 'bri') }}
+{%                 if rule_conf.recent is vyos_defined %}
+{%                     set ns.sets = ns.sets + ['OUT_' + prior + '_' + rule_id] %}
+{%                 endif %}
+{%             endfor %}
+{%         endif %}
+    {{ conf | nft_default_rule('OUT-filter', 'bri') }}
+    }
+{%     endfor %}
+{% endif %}
+
 {% if bridge.name is vyos_defined %}
 {%     for name_text, conf in bridge.name.items() %}
     chain NAME_{{ name_text }} {
@@ -32,4 +76,45 @@
     }
 {%     endfor %}
 {% endif %}
+
+{% for set_name in ns.sets %}
+    set RECENT_{{ set_name }} {
+        type ipv4_addr
+        size 65535
+        flags dynamic
+    }
+{% endfor %}
+{% for set_name in ip_fqdn %}
+    set FQDN_{{ set_name }} {
+        type ipv4_addr
+        flags interval
+    }
+{% endfor %}
+{% if geoip_updated.name is vyos_defined %}
+{%     for setname in geoip_updated.name %}
+    set {{ setname }} {
+        type ipv4_addr
+        flags interval
+    }
+{%     endfor %}
+{% endif %}
+
+{{ group_tmpl.groups(group, False, True) }}
+{{ group_tmpl.groups(group, True, True) }}
+
+{% if global_options.state_policy is vyos_defined %}
+    chain VYOS_STATE_POLICY {
+{%     if global_options.state_policy.established is vyos_defined %}
+        {{ global_options.state_policy.established | nft_state_policy('established') }}
+{%     endif %}
+{%     if global_options.state_policy.invalid is vyos_defined %}
+        {{ global_options.state_policy.invalid | nft_state_policy('invalid') }}
+{%     endif %}
+{%     if global_options.state_policy.related is vyos_defined %}
+        {{ global_options.state_policy.related | nft_state_policy('related') }}
+{%     endif %}
+        return
+    }
+{% endif %}
+
 {% endmacro %}
diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2
@@ -1,7 +1,7 @@
 {% macro groups(group, is_ipv6, is_l3) %}
 {% if group is vyos_defined %}
 {%     set ip_type = 'ipv6_addr' if is_ipv6 else 'ipv4_addr' %}
-{%     if group.address_group is vyos_defined and not is_ipv6 and is_l3 %}
+{%     if group.address_group is vyos_defined and not is_ipv6 %}
 {%         for group_name, group_conf in group.address_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
     set A_{{ group_name }} {
@@ -14,7 +14,7 @@
     }
 {%         endfor %}
 {%     endif %}
-{%     if group.ipv6_address_group is vyos_defined and is_ipv6 and is_l3 %}
+{%     if group.ipv6_address_group is vyos_defined and is_ipv6 %}
 {%         for group_name, group_conf in group.ipv6_address_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
     set A6_{{ group_name }} {
@@ -46,7 +46,7 @@
     }
 {%         endfor %}
 {%     endif %}
-{%     if group.network_group is vyos_defined and not is_ipv6 and is_l3 %}
+{%     if group.network_group is vyos_defined and not is_ipv6 %}
 {%         for group_name, group_conf in group.network_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
     set N_{{ group_name }} {
@@ -59,7 +59,7 @@
     }
 {%         endfor %}
 {%     endif %}
-{%     if group.ipv6_network_group is vyos_defined and is_ipv6 and is_l3 %}
+{%     if group.ipv6_network_group is vyos_defined and is_ipv6 %}
 {%         for group_name, group_conf in group.ipv6_network_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
     set N6_{{ group_name }} {
@@ -72,7 +72,7 @@
     }
 {%         endfor %}
 {%     endif %}
-{%     if group.port_group is vyos_defined and is_l3 %}
+{%     if group.port_group is vyos_defined %}
 {%         for group_name, group_conf in group.port_group.items() %}
 {%             set includes = group_conf.include if group_conf.include is vyos_defined else [] %}
     set P_{{ group_name }} {

diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2
@@ -135,7 +135,7 @@ table ip vyos_filter {
 {%                     endif %}
 {%                 endfor %}
 {%             endif %}
-        {{ conf | nft_default_rule(name_text, 'ipv4') }}
+        {{ conf | nft_default_rule('NAM-' + name_text, 'ipv4') }}
     }
 {%         endfor %}
 {%     endif %}
@@ -287,7 +287,7 @@ table ip6 vyos_filter {
 {%                     endif %}
 {%                 endfor %}
 {%             endif %}
-        {{ conf | nft_default_rule(name_text, 'ipv6') }}
+        {{ conf | nft_default_rule('NAM-' + name_text, 'ipv6') }}
     }
 {%         endfor %}
 {%     endif %}
@@ -339,7 +339,104 @@ table ip6 vyos_filter {
 delete table bridge vyos_filter
 {% endif %}
 table bridge vyos_filter {
-{{ bridge_tmpl.bridge(bridge) }}
+{% if bridge is vyos_defined %}
+{%     if bridge.forward is vyos_defined %}
+{%         for prior, conf in bridge.forward.items() %}
+    chain VYOS_FORWARD_{{ prior }} {
+        type filter hook forward priority {{ prior }}; policy accept;
+{%             if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%             endif %}
+{%             if conf.rule is vyos_defined %}
+{%                 for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+        {{ rule_conf | nft_rule('FWD', prior, rule_id, 'bri') }}
+{%                 endfor %}
+{%             endif %}
+        {{ conf | nft_default_rule('FWD-' + prior, 'bri') }}
+    }
+{%         endfor %}
+{%     endif %}
+
+{%     if bridge.input is vyos_defined %}
+{%         for prior, conf in bridge.input.items() %}
+    chain VYOS_INPUT_{{ prior }} {
+        type filter hook input priority {{ prior }}; policy accept;
+{%             if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%             endif %}
+{%             if conf.rule is vyos_defined %}
+{%                 for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+        {{ rule_conf | nft_rule('INP', prior, rule_id, 'bri') }}
+{%                 endfor %}
+{%             endif %}
+        {{ conf | nft_default_rule('INP-' + prior, 'bri') }}
+    }
+{%         endfor %}
+{%     endif %}
+
+{%     if bridge.output is vyos_defined %}
+{%         for prior, conf in bridge.output.items() %}
+    chain VYOS_OUTUT_{{ prior }} {
+        type filter hook output priority {{ prior }}; policy accept;
+{%             if global_options.state_policy is vyos_defined %}
+        jump VYOS_STATE_POLICY
+{%             endif %}
+{%             if conf.rule is vyos_defined %}
+{%                 for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+        {{ rule_conf | nft_rule('OUT', prior, rule_id, 'bri') }}
+{%                 endfor %}
+{%             endif %}
+        {{ conf | nft_default_rule('OUT-' + prior, 'bri') }}
+    }
+{%         endfor %}
+{%     endif %}
+
+{%     if bridge.prerouting is vyos_defined %}
+{%         for prior, conf in bridge.prerouting.items() %}
+    chain VYOS_PREROUTING_{{ prior }} {
+        type filter hook prerouting priority {{ prior }}; policy accept;
+{%             if conf.rule is vyos_defined %}
+{%                 for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+        {{ rule_conf | nft_rule('PRE', prior, rule_id, 'bri') }}
+{%                 endfor %}
+{%             endif %}
+        {{ conf | nft_default_rule('PRE-' + prior, 'bri') }}
+    }
+{%         endfor %}
+{%     endif %}
+
+{%     if bridge.name is vyos_defined %}
+{%         for name_text, conf in bridge.name.items() %}
+    chain NAME_{{ name_text }} {
+{%             if conf.rule is vyos_defined %}
+{%                 for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %}
+        {{ rule_conf | nft_rule('NAM', name_text, rule_id, 'bri') }}
+{%                     if rule_conf.recent is vyos_defined %}
+{%                         set ns.sets = ns.sets + ['NAM_' + name_text + '_' + rule_id] %}
+{%                     endif %}
+{%                 endfor %}
+{%             endif %}
+        {{ conf | nft_default_rule('NAM-' + name_text, 'bri') }}
+    }
+{%         endfor %}
+{%     endif %}
+
+{% endif %}
 {{ group_tmpl.groups(group, False, False) }}
+{{ group_tmpl.groups(group, True, False) }}
 
-}
+{% if global_options.state_policy is vyos_defined %}
+    chain VYOS_STATE_POLICY {
+{%     if global_options.state_policy.established is vyos_defined %}
+        {{ global_options.state_policy.established | nft_state_policy('established') }}
+{%     endif %}
+{%     if global_options.state_policy.invalid is vyos_defined %}
+        {{ global_options.state_policy.invalid | nft_state_policy('invalid') }}
+{%     endif %}
+{%     if global_options.state_policy.related is vyos_defined %}
+        {{ global_options.state_policy.related | nft_state_policy('related') }}
+{%     endif %}
+        return
+    }
+{% endif %}
+}