KVM: add configurable MAC/IP script hook for static ARP/NDP and routes

Introduces a new agent.properties option `vm.network.macip.script`
(absolute path, disabled by default) that BridgeVifDriver invokes on
every NIC plug (VM start) and unplug (VM stop).

This is very useful in EVPN+VXLAN environments as it can reduce BUM
traffic. By setting static ARP/NDP entries bridges can be configured
using 'neigh_suppress on' as the ARP/NDP entries are already set
statically by CloudStack.

Author: wido

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -914,6 +914,20 @@ public Property<Integer> getWorkers() {
      * */
     public static final Property<Integer> INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60);
 
+    /**
+     * Absolute path to a script that is executed on VM NIC plug (VM start) and unplug (VM stop)
+     * to manage static ARP/NDP entries and host routes for VM interfaces.<br>
+     * The script is invoked with:<br>
+     * &nbsp;&nbsp;add:    <code>-o add -b &lt;bridge&gt; -m &lt;mac&gt; [-4 &lt;ipv4&gt;] [-6 &lt;ipv6&gt;]</code><br>
+     * &nbsp;&nbsp;delete: <code>-o delete -b &lt;bridge&gt; -m &lt;mac&gt;</code><br>
+     * A bundled reference implementation is available at
+     * <code>scripts/vm/network/vnet/modifymacip.sh</code>.<br>
+     * Leave empty or unset to disable this feature.<br>
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> VM_NETWORK_MACIP_SCRIPT = new Property<>("vm.network.macip.script", null, String.class);
+
 
     public static class Property <T>{
         private String name;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java

@@ -48,6 +48,7 @@ public class BridgeVifDriver extends VifDriverBase {
     private final Object _vnetBridgeMonitor = new Object();
     private String _modifyVlanPath;
     private String _modifyVxlanPath;
+    private String _macIpScriptPath;
     private String _controlCidr = NetUtils.getLinkLocalCIDR();
     private Long libvirtVersion;
 
@@ -81,6 +82,12 @@ public void configure(Map<String, Object> params) throws ConfigurationException
             throw new ConfigurationException("Unable to find modifyvxlan.sh");
         }
 
+        String macIpScript = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_SCRIPT);
+        if (macIpScript != null && !macIpScript.isEmpty()) {
+            _macIpScriptPath = macIpScript;
+            logger.info("VM network MAC/IP script configured: {}", _macIpScriptPath);
+        }
+
         libvirtVersion = (Long) params.get("libvirtVersion");
         if (libvirtVersion == null) {
             libvirtVersion = 0L;
@@ -276,11 +283,14 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA
             intf.setPxeDisable(true);
         }
 
+        executeMacIpScript("add", intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address());
+
         return intf;
     }
 
     @Override
     public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) {
+        executeMacIpScript("delete", iface.getBrName(), iface.getMacAddress(), null, null);
         deleteVnetBr(iface.getBrName(), deleteBr);
     }
 
@@ -400,6 +410,26 @@ private void deleteVnetBr(String brName, boolean deleteBr) {
         }
     }
 
+    private void executeMacIpScript(String op, String brName, String mac, String ipv4, String ipv6) {
+        if (_macIpScriptPath == null || mac == null || brName == null) {
+            return;
+        }
+        final Script command = new Script(_macIpScriptPath, _timeout, logger);
+        command.add("-o", op);
+        command.add("-b", brName);
+        command.add("-m", mac);
+        if (ipv4 != null && !ipv4.isEmpty()) {
+            command.add("-4", ipv4);
+        }
+        if (ipv6 != null && !ipv6.isEmpty()) {
+            command.add("-6", ipv6);
+        }
+        final String result = command.execute();
+        if (result != null) {
+            logger.warn("MAC/IP script returned error for {} on {}: {}", op, mac, result);
+        }
+    }
+
     private void deleteExistingLinkLocalRouteTable(String linkLocalBr) {
         Script command = new Script("/bin/bash", _timeout);
         command.add("-c");

scripts/vm/network/vnet/modifymacip.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# modifymacip.sh -- Manage static ARP/NDP entries and host routes for VM NICs
+#
+# Usage:
+#   add:    modifymacip.sh -o add    -b <bridge> -m <mac> [-4 <ipv4>] [-6 <ipv6>]
+#   delete: modifymacip.sh -o delete -b <bridge> -m <mac>
+#
+# On add the script persists the NIC's address info so that delete can
+# recover the IPs without them being passed explicitly.
+
+STATE_DIR=/var/run/cloud/macip
+
+usage() {
+    echo "Usage: $0 -o <add|delete> -b <bridge> -m <mac> [-4 <ipv4>] [-6 <ipv6>]"
+}
+
+OP=
+BRIDGE=
+MAC=
+IPV4=
+IPV6=
+
+while getopts 'o:b:m:4:6:' OPTION; do
+    case $OPTION in
+    o) OP="$OPTARG" ;;
+    b) BRIDGE="$OPTARG" ;;
+    m) MAC="$OPTARG" ;;
+    4) IPV4="$OPTARG" ;;
+    6) IPV6="$OPTARG" ;;
+    ?) usage; exit 2 ;;
+    esac
+done
+
+if [[ -z "$OP" || -z "$BRIDGE" || -z "$MAC" ]]; then
+    usage
+    exit 2
+fi
+
+# Normalise MAC address for use as a filename (replace colons with underscores)
+MAC_NORM="${MAC//:/_}"
+STATE_FILE="${STATE_DIR}/${MAC_NORM}.conf"
+
+add_entries() {
+    mkdir -p "${STATE_DIR}"
+
+    # Persist state so that delete can recover IPs without them being passed in
+    printf 'IPV4=%s\nIPV6=%s\nBRIDGE=%s\n' "${IPV4}" "${IPV6}" "${BRIDGE}" > "${STATE_FILE}"
+
+    if [[ -n "$IPV4" ]]; then
+        ip neigh replace "${IPV4}" lladdr "${MAC}" dev "${BRIDGE}" nud permanent
+        ip route replace "${IPV4}/32" dev "${BRIDGE}"
+    fi
+
+    if [[ -n "$IPV6" ]]; then
+        ip -6 neigh replace "${IPV6}" lladdr "${MAC}" dev "${BRIDGE}" nud permanent
+        ip -6 route replace "${IPV6}/128" dev "${BRIDGE}"
+    fi
+}
+
+delete_entries() {
+    local del_ipv4="${IPV4}"
+    local del_ipv6="${IPV6}"
+    local del_bridge="${BRIDGE}"
+
+    # Recover IPs and bridge from the state file written at add time
+    if [[ -f "${STATE_FILE}" ]]; then
+        while IFS='=' read -r key value; do
+            case "$key" in
+                IPV4)   del_ipv4="${del_ipv4:-$value}" ;;
+                IPV6)   del_ipv6="${del_ipv6:-$value}" ;;
+                BRIDGE) del_bridge="${del_bridge:-$value}" ;;
+            esac
+        done < "${STATE_FILE}"
+    fi
+
+    if [[ -n "$del_ipv4" ]]; then
+        ip neigh del "${del_ipv4}" dev "${del_bridge}" 2>/dev/null || true
+        ip route del "${del_ipv4}/32" dev "${del_bridge}" 2>/dev/null || true
+    fi
+
+    if [[ -n "$del_ipv6" ]]; then
+        ip -6 neigh del "${del_ipv6}" dev "${del_bridge}" 2>/dev/null || true
+        ip -6 route del "${del_ipv6}/128" dev "${del_bridge}" 2>/dev/null || true
+    fi
+
+    rm -f "${STATE_FILE}"
+}
+
+case "$OP" in
+    add)    add_entries ;;
+    delete) delete_entries ;;
+    *)      usage; exit 2 ;;
+esac