diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index 6c362163c3..22e066def5 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -59,6 +59,7 @@ class Ethtool: _ring_buffer = None _driver_name = None _flow_control = None + _channels = '' def __init__(self, ifname): # Get driver used for interface @@ -111,6 +112,11 @@ def __init__(self, ifname): if not bool(err): self._flow_control = loads(out)[0] + # Get information about NIC channels + out, err = popen(f'ethtool --show-channels {ifname}') + if not bool(err): + self._channels = out.lower() + def check_auto_negotiation_supported(self): """ Check if the NIC supports changing auto-negotiation """ return self._base_settings['supports-auto-negotiation'] @@ -199,3 +205,20 @@ def get_flow_control(self): 'flow-control settings!') return 'on' if bool(self._flow_control['autonegotiate']) else 'off' + + def get_channels(self, rx_tx_comb): + """ + Get both the pre-set maximum and current value for a given channel type. + + Args: + rx_tx_comb (str): Channel type, one of "rx", "tx", or "combined". + + Returns: + list[int]: [maximum, current] values for the channel, + or an empty list if not supported. + """ + if rx_tx_comb not in ['rx', 'tx', 'combined']: + raise ValueError('Channel type must be either "rx", "tx" or "combined"') + matches = re.findall(rf'{rx_tx_comb}:\s+(\d+)', self._channels) + + return [int(value) for value in matches] diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 6c7b230ca0..70da837b9d 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -455,6 +455,23 @@ def set_ring_buffer(self, rx_tx, size): print(f'could not set "{rx_tx}" ring-buffer for {ifname}') return output + def set_channels(self, rx_tx_comb, queues): + """ + Example: + >>> from vyos.ifconfig import EthernetIf + >>> i = EthernetIf('eth0') + >>> i.set_channels('rx', 2) + """ + ifname = self.config['ifname'] + cmd = f'ethtool --set-channels {ifname} {rx_tx_comb} {queues}' + output, code = self._popen(cmd) + # ethtool error codes: + # 80 - value already setted + # 81 - does not possible to set value + if code and code != 80: + print(f'could not set "{rx_tx_comb}" channel for {ifname}') + return output + def set_switchdev(self, enable): ifname = self.config['ifname'] addr, code = self._popen( diff --git a/python/vyos/vpp/control_host.py b/python/vyos/vpp/control_host.py index 6a6b9d72f4..ea40e6b75e 100644 --- a/python/vyos/vpp/control_host.py +++ b/python/vyos/vpp/control_host.py @@ -23,6 +23,8 @@ from pyroute2 import IPRoute +from vyos.ethtool import Ethtool +from vyos.ifconfig import EthernetIf from vyos.vpp.utils import EthtoolGDrvinfo @@ -373,3 +375,40 @@ def flush_ip(iface_name: str) -> None: """ iproute = IPRoute() iproute.flush_addr(label=iface_name) + + +def get_eth_channels(iface_name: str) -> dict: + """ + Get the current hardware queue counts for channels of an interface using ethtool. + + Args: + iface_name (str): name of an interface + + Returns: + dict: Mapping of channel types to their current values: + - 'rx' (int | None): RX channel count. + - 'tx' (int | None): TX channel count. + - 'combined' (int | None): Combined channel count. + Returns None if the channel type is not supported. + """ + ethtool = Ethtool(iface_name) + + channels = {} + for channel in ['rx', 'tx', 'combined']: + queues_list = ethtool.get_channels(channel) + channels[channel] = queues_list[-1] if (len(queues_list) == 2) else None + + return channels + + +def set_eth_channels(iface_name: str, channels: dict) -> None: + """Configure the number of RX, TX, or combined channels for an interface. + + Args: + iface_name (str): name of an interface + channels (dict): channels to set + """ + interface = EthernetIf(iface_name) + for channel, value in channels.items(): + if value: + interface.set_channels(channel, value) diff --git a/python/vyos/vpp/control_vpp.py b/python/vyos/vpp/control_vpp.py index 42e7bf6c12..ff291e96f6 100644 --- a/python/vyos/vpp/control_vpp.py +++ b/python/vyos/vpp/control_vpp.py @@ -351,7 +351,7 @@ def xdp_iface_create( self, host_if: str, name: str, - rxq_num: int = 0, + rxq_num: int = 65535, rxq_size: int = 0, txq_size: int = 0, mode: Literal['auto', 'copy', 'zero-copy'] = 'auto', diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index 492fa92da3..ae6a3731e0 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -112,6 +112,9 @@ 'virtio_net': ['xdp'], } +# drivers that require changing channels (half the maximum number of RX/TX queues) +ethtool_channels_change_drv: list[str] = ['ena', 'gve'] + def _load_module(module_name: str): """ @@ -171,6 +174,25 @@ def _normalize_buffers(config: dict): config['settings']['buffers']['buffers_per_numa'] = str(buffers) +def _get_max_xdp_rx_queues(config: dict): + """ + Count max number of RX queues for XDP driver + - If the interface driver is in `ethtool_channels_change_drv` + only half of the available queues are used (to avoid NIC issues) + - For other interface drivers the full number of queues is returned. + - If neither `rx` nor `combined` is set, return 1. + """ + for key in ('rx', 'combined'): + value = config['channels'].get(key) + if value: + if config['original_driver'] in ethtool_channels_change_drv: + return max(1, int(value) // 2) + else: + return int(value) + + return 1 + + def get_config(config=None): # use persistent config to store interfaces data between executions # this is required because some interfaces after they are connected @@ -284,6 +306,20 @@ def get_config(config=None): _normalize_buffers(effective_config) config['effective'] = effective_config + # Save important info about all interfaces that cannot be retrieved later + # Add new interfaces (only if they are first time seen in a config) + for iface, iface_config in config.get('settings', {}).get('interface', {}).items(): + if iface not in effective_config.get('settings', {}).get('interface', {}): + eth_ifaces_persist[iface] = { + 'original_driver': EthtoolGDrvinfo(iface).driver, + } + eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface) + eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface) + eth_ifaces_persist[iface]['channels'] = control_host.get_eth_channels(iface) + + # Return to config dictionary + config['persist_config'] = eth_ifaces_persist + if 'settings' in config: if 'interface' in config['settings']: for iface, iface_config in config['settings']['interface'].items(): @@ -307,11 +343,11 @@ def get_config(config=None): iface_filter_eth(conf, iface) set_dependents('ethernet', conf, iface) # Interfaces with changed driver should be removed/readded - if old_driver and old_driver[0] == 'dpdk': + if old_driver: removed_ifaces.append( { 'iface_name': iface, - 'driver': 'dpdk', + 'driver': old_driver[0], } ) @@ -344,7 +380,8 @@ def get_config(config=None): 'txq_size': int(iface_config['xdp_options']['tx_queue_size']), } if iface_config['xdp_options']['num_rx_queues'] == 'all': - xdp_api_params['rxq_num'] = 0 + # 65535 is used as special value to request all available queues + xdp_api_params['rxq_num'] = 65535 else: xdp_api_params['rxq_num'] = int( iface_config['xdp_options']['num_rx_queues'] @@ -374,26 +411,11 @@ def get_config(config=None): iface_filter_eth(conf, iface) set_dependents(dependency, conf, iface) - # Save important info about all interfaces that cannot be retrieved later - # Add new interfaces (only if they are first time seen in a config) - for iface, iface_config in config.get('settings', {}).get('interface', {}).items(): - if iface not in effective_config.get('settings', {}).get('interface', {}): - eth_ifaces_persist[iface] = { - 'original_driver': config['settings']['interface'][iface][ - 'kernel_module' - ], - } - eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface) - eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface) - # PPPoE dependency if pppoe_map_ifaces: config['pppoe_ifaces'] = pppoe_map_ifaces set_dependents('pppoe_server', conf) - # Return to config dictionary - config['persist_config'] = eth_ifaces_persist - return config @@ -485,6 +507,14 @@ def verify(config): ) if iface_config['driver'] == 'xdp' and 'xdp_options' in iface_config: if iface_config['xdp_options']['num_rx_queues'] != 'all': + rx_queues = iface_config['xdp_api_params']['rxq_num'] + max_rx_queues = _get_max_xdp_rx_queues(config['persist_config'][iface]) + if rx_queues > max_rx_queues: + raise ConfigError( + f'Maximum supported number of RX queues for interface {iface} is {max_rx_queues}. ' + f'Please set "xdp-options num-rx-queues" to {max_rx_queues} or fewer' + ) + Warning(f'Not all RX queues will be connected to VPP for {iface}!') if iface_config['driver'] == 'xdp' and 'dpdk_options' in iface_config: @@ -593,8 +623,10 @@ def initialize_interface(iface, driver, iface_config) -> None: iface_new_name: str = control_host.get_eth_name(iface_config['dev_id']) control_host.rename_iface(iface_new_name, iface) - # XDP - rename an interface, disable promisc and XDP + # XDP - rename an interface, set original channels, disable promisc and XDP if driver == 'xdp': + if iface_config['original_driver'] in ethtool_channels_change_drv: + control_host.set_eth_channels(f'defunct_{iface}', iface_config['channels']) control_host.set_promisc(f'defunct_{iface}', 'off') control_host.rename_iface(f'defunct_{iface}', iface) control_host.xdp_remove(iface) @@ -691,6 +723,25 @@ def apply(config): # add XDP interfaces if iface_config['driver'] == 'xdp': control_host.rename_iface(iface, f'defunct_{iface}') + + # Some cloud NICs fail to load XDP if all RX queues are configured. To avoid this, + # we limit the number of queues to half of the maximum supported by the driver. + if ( + config['persist_config'][iface]['original_driver'] + in ethtool_channels_change_drv + ): + max_rx_queues = _get_max_xdp_rx_queues( + config['persist_config'][iface] + ) + channels_orig = config['persist_config'][iface]['channels'] + channels = {} + if channels_orig.get('rx'): + channels = {'rx': max_rx_queues, 'tx': max_rx_queues} + if channels_orig.get('combined'): + channels['combined'] = max_rx_queues + if channels: + control_host.set_eth_channels(f'defunct_{iface}', channels) + vpp_control.xdp_iface_create( host_if=f'defunct_{iface}', name=iface,