Stop flooding other guests' unicast frames to isolated TAPs

lib/network/allocate.go

@@ -59,7 +59,7 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N
 		attribute.Bool("download_rate_limit", req.DownloadBps > 0),
 		attribute.Bool("upload_rate_limit", req.UploadBps > 0),
 	)
-	err = m.createTAPDevice(tapCtx, netConfig.TAPDevice, network.Bridge, network.Isolated)
+	err = m.createTAPDevice(tapCtx, netConfig.TAPDevice, network.Bridge, netConfig.MAC, network.Isolated)
 	tapSpanEnd(err)
 	if err != nil {
 		cleanupErr := m.deleteTAPDeviceForInstanceSerialized(ctx, req.InstanceID, netConfig.TAPDevice)
@@ -143,7 +143,7 @@ func (m *manager) RecreateAllocation(ctx context.Context, instanceID string, dow
 		attribute.Bool("download_rate_limit", downloadBps > 0),
 		attribute.Bool("upload_rate_limit", uploadBps > 0),
 	)
-	err = m.createTAPDevice(tapCtx, alloc.TAPDevice, network.Bridge, network.Isolated)
+	err = m.createTAPDevice(tapCtx, alloc.TAPDevice, network.Bridge, alloc.MAC, network.Isolated)
 	tapSpanEnd(err)
 	if err != nil {
 		_ = m.deleteTAPDeviceForInstanceSerialized(ctx, instanceID, alloc.TAPDevice)

lib/network/bridge_darwin.go

@@ -36,7 +36,7 @@ func (m *manager) setupBridgeHTB(ctx context.Context, bridgeName string, capacit
 
 // createTAPDevice is a no-op on macOS as we use NAT networking.
 // Virtualization.framework creates virtual network interfaces internally.
-func (m *manager) createTAPDevice(ctx context.Context, tapName, bridgeName string, isolated bool) error {
+func (m *manager) createTAPDevice(ctx context.Context, tapName, bridgeName, guestMAC string, isolated bool) error {
 	// On macOS with vz, network devices are created by the VMM itself
 	return nil
 }

lib/network/bridge_linux.go

@@ -526,7 +526,16 @@ func (m *manager) lastHypemanForwardRulePosition() int {
 }
 
 // createTAPDevice creates TAP device and attaches it to the bridge.
-func (m *manager) createTAPDevice(ctx context.Context, tapName, bridgeName string, isolated bool) error {
+func (m *manager) createTAPDevice(ctx context.Context, tapName, bridgeName, guestMAC string, isolated bool) error {
+	var mac net.HardwareAddr
+	if isolated {
+		var err error
+		mac, err = net.ParseMAC(guestMAC)
+		if err != nil {
+			return fmt.Errorf("parse guest MAC %q: %w", guestMAC, err)
+		}
+	}
+
 	// 1. Check if TAP already exists
 	_, linkLookupEnd := startNetworkStep(ctx, "network.create_tap.link_lookup_existing",
 		attribute.String("operation", "link_lookup_existing"),
@@ -620,11 +629,51 @@ func (m *manager) createTAPDevice(ctx context.Context, tapName, bridgeName strin
 		if err != nil {
 			return fmt.Errorf("set isolation mode: %w", err)
 		}
+
+		// 6. Isolation only stops guest-to-guest forwarding. Frames for a MAC the
+		// bridge has no entry for are still flooded to every port, so a guest
+		// in promiscuous mode can read other guests' inbound traffic. Pin the
+		// guest's MAC to its port, then stop flooding and learning on it.
+		_, hardenEnd := startNetworkStep(ctx, "network.create_tap.harden_port",
+			attribute.String("operation", "harden_port"),
+			attribute.String("tap", tapName),
+		)
+		err = hardenIsolatedPort(tapLink, mac)
+		hardenEnd(err)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
+// hardenIsolatedPort pins mac to the bridge port as a static FDB entry and
+// turns off unknown-unicast flooding and source-MAC learning on the port.
+// Learning must be off too: a frame with a spoofed source MAC moves even a
+// static entry to the sending port.
+func hardenIsolatedPort(port netlink.Link, mac net.HardwareAddr) error {
+	// NUD_NOARP is `bridge fdb add ... static`. NUD_PERMANENT would make the
+	// entry local to the bridge instead of an address behind this port.
+	err := netlink.NeighSet(&netlink.Neigh{
+		LinkIndex:    port.Attrs().Index,
+		Family:       unix.AF_BRIDGE,
+		State:        netlink.NUD_NOARP,
+		Flags:        netlink.NTF_MASTER,
+		HardwareAddr: mac,
+	})
+	if err != nil {
+		return fmt.Errorf("pin guest MAC in bridge FDB: %w", err)
+	}
+	if err := netlink.LinkSetFlood(port, false); err != nil {
+		return fmt.Errorf("disable unicast flooding: %w", err)
+	}
+	if err := netlink.LinkSetLearning(port, false); err != nil {
+		return fmt.Errorf("disable MAC learning: %w", err)
+	}
+	return nil
+}
+
 // applyDownloadRateLimit applies download (external→VM) rate limiting using TBF on TAP egress.
 func (m *manager) applyDownloadRateLimit(ctx context.Context, tapName string, rateLimitBps int64) error {
 	rateStr := formatTcRate(rateLimitBps)

lib/network/harden_port_linux_test.go

@@ -0,0 +1,172 @@
+//go:build linux
+
+package network
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+const testEtherType = 0x88b5
+
+// TestHardenIsolatedPortStopsUnicastLeak builds a bridge in a throwaway network
+// namespace with two hardened guest ports and one port left at kernel defaults,
+// then transmits frames through the bridge the way the host stack does.
+func TestHardenIsolatedPortStopsUnicastLeak(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("Skipping test that requires root")
+	}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	orig, err := netns.Get()
+	require.NoError(t, err)
+	defer orig.Close()
+	ns, err := netns.New()
+	require.NoError(t, err)
+	defer ns.Close()
+	defer netns.Set(orig)
+
+	require.NoError(t, netlink.LinkAdd(&netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: "br0"}}))
+	br, err := netlink.LinkByName("br0")
+	require.NoError(t, err)
+	require.NoError(t, netlink.LinkSetUp(br))
+
+	guestA := addBridgePort(t, br, "a", false)
+	guestB := addBridgePort(t, br, "b", false)
+	control := addBridgePort(t, br, "c", true)
+
+	waitUp(t, "br0", "port-a", "port-b", "port-c", "guest-a", "guest-b", "guest-c")
+	recvA := openPacketSocket(t, guestA)
+	recvB := openPacketSocket(t, guestB)
+	recvControl := openPacketSocket(t, control)
+	send := openPacketSocket(t, br)
+
+	macA := guestA.Attrs().HardwareAddr
+	unknown := net.HardwareAddr{0x02, 0x00, 0x00, 0xde, 0xad, 0x01}
+
+	// Delivery to a pinned guest must not depend on flooding.
+	sendFrame(t, send, br, macA, []byte("to-a"))
+	assert.True(t, receivedFrame(recvA, []byte("to-a")), "pinned guest did not receive its frame")
+	assert.False(t, receivedFrame(recvB, []byte("to-a")), "frame for guest A reached guest B")
+
+	// Unknown unicast reaches a default port (the leak) but not hardened ones.
+	sendFrame(t, send, br, unknown, []byte("unknown"))
+	assert.True(t, receivedFrame(recvControl, []byte("unknown")), "control port should see flooded unknown unicast")
+	assert.False(t, receivedFrame(recvA, []byte("unknown")), "unknown unicast flooded to hardened guest A")
+	assert.False(t, receivedFrame(recvB, []byte("unknown")), "unknown unicast flooded to hardened guest B")
+
+	// Guest B spoofing guest A's MAC must not move A's FDB entry.
+	sendFrame(t, recvB, guestB, net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, []byte("spoof"), macA)
+	sendFrame(t, send, br, macA, []byte("after-spoof"))
+	assert.True(t, receivedFrame(recvA, []byte("after-spoof")), "guest A lost delivery after spoof")
+	assert.False(t, receivedFrame(recvB, []byte("after-spoof")), "spoofing guest captured guest A's frame")
+}
+
+// addBridgePort creates a veth pair, enslaves one end to br as an isolated port
+// and returns the other end, which stands in for the guest NIC.
+func addBridgePort(t *testing.T, br netlink.Link, name string, defaults bool) netlink.Link {
+	t.Helper()
+	veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "port-" + name}, PeerName: "guest-" + name}
+	require.NoError(t, netlink.LinkAdd(veth))
+	port, err := netlink.LinkByName("port-" + name)
+	require.NoError(t, err)
+	guest, err := netlink.LinkByName("guest-" + name)
+	require.NoError(t, err)
+	require.NoError(t, netlink.LinkSetMaster(port, br))
+	require.NoError(t, netlink.LinkSetIsolated(port, true))
+	if !defaults {
+		require.NoError(t, hardenIsolatedPort(port, guest.Attrs().HardwareAddr))
+
+		neighs, err := netlink.NeighList(port.Attrs().Index, unix.AF_BRIDGE)
+		require.NoError(t, err)
+		var pinned *netlink.Neigh
+		for i := range neighs {
+			if bytes.Equal(neighs[i].HardwareAddr, guest.Attrs().HardwareAddr) {
+				pinned = &neighs[i]
+			}
+		}
+		require.NotNil(t, pinned, "guest MAC not in bridge FDB")
+		assert.Equal(t, netlink.NUD_NOARP, pinned.State, "FDB entry should be static, not local")
+	}
+	require.NoError(t, netlink.LinkSetUp(port))
+	require.NoError(t, netlink.LinkSetUp(guest))
+	return guest
+}
+
+// waitUp waits for the named links to come up. Bridge ports with STP off start
+// forwarding once they have carrier. It polls on the calling goroutine because
+// the links only exist in this thread's network namespace.
+func waitUp(t *testing.T, names ...string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for _, name := range names {
+		for {
+			link, err := netlink.LinkByName(name)
+			require.NoError(t, err)
+			if link.Attrs().OperState == netlink.OperUp {
+				break
+			}
+			require.True(t, time.Now().Before(deadline), "%s did not come up", name)
+			time.Sleep(20 * time.Millisecond)
+		}
+	}
+}
+
+func openPacketSocket(t *testing.T, link netlink.Link) int {
+	t.Helper()
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(testEtherType)))
+	require.NoError(t, err)
+	t.Cleanup(func() { unix.Close(fd) })
+	require.NoError(t, unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(testEtherType), Ifindex: link.Attrs().Index}))
+	tv := unix.NsecToTimeval((200 * time.Millisecond).Nanoseconds())
+	require.NoError(t, unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv))
+	return fd
+}
+
+func sendFrame(t *testing.T, fd int, link netlink.Link, dst net.HardwareAddr, payload []byte, src ...net.HardwareAddr) {
+	t.Helper()
+	from := link.Attrs().HardwareAddr
+	if len(src) > 0 {
+		from = src[0]
+	}
+	frame := append(append(append([]byte{}, dst...), from...), byte(testEtherType>>8), byte(testEtherType&0xff))
+	frame = append(frame, payload...)
+	for len(frame) < 60 {
+		frame = append(frame, 0)
+	}
+	require.NoError(t, unix.Sendto(fd, frame, 0, &unix.SockaddrLinklayer{Protocol: htons(testEtherType), Ifindex: link.Attrs().Index, Halen: 6}))
+}
+
+// receivedFrame reports whether a frame carrying payload arrives before the
+// socket's read timeout.
+func receivedFrame(fd int, payload []byte) bool {
+	buf := make([]byte, 2048)
+	for {
+		n, from, err := unix.Recvfrom(fd, buf, 0)
+		if err == unix.EINTR {
+			continue
+		}
+		if err != nil {
+			return false
+		}
+		if ll, ok := from.(*unix.SockaddrLinklayer); ok && ll.Pkttype == unix.PACKET_OUTGOING {
+			continue
+		}
+		if n > 14 && bytes.HasPrefix(buf[14:n], payload) {
+			return true
+		}
+	}
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }