Align the default route spec with FRR

- Use unreachable route instead of blackhole. This allows Linux to send
  ICMP unreachable instead of silently discard the route.
- Set metric to 4278198272 instead of 100. This is based on the special
  metrics encoding used by FRR which interprets upper 1 byte as AD and
  lower 3 bytes as an actual metric.

Signed-off-by: Yutaro Hayakawa <[email protected]>

hostvrf.go

@@ -200,25 +200,34 @@ func ensureLoopbackAddresses(n *NetConf, vrf *netlink.Vrf) error {
 	return nil
 }
 
-// ensureBlackholeRoutes inserts blackhole routes to the VRF device.
-// This route is used to isolate the traffic from the VRF device to
-// the outside world by default. In Linux, VRF is implemented as an
-// IP rule which is evaluated earlier than main routing table. Thus,
-// without this blackhole route, the routing lookup falls through
-// to the main table and packets hits the routing entries there.
+// ensureUnreachableDefaultRoutes inserts unreachable routes to the VRF device.
+// This route is used to isolate the traffic from the VRF device to the outside
+// world by default. In Linux, VRF is implemented as an IP rule which is
+// evaluated earlier than main routing table. Thus, without this unreachable
+// route, the routing lookup falls through to the main table and packets hits
+// the routing entries there.
 //
 // If users wish to direct traffic to other VRFs, they can "leak" the routes
 // from other VRFs. They can even override the default route by inserting the
-// default route with the priority higher than 100.
-func ensureBlackholeRoutes(n *NetConf, vrf *netlink.Vrf) error {
+// default route with the priority (metric) lower than 4278198272.
+//
+// This weird priority value is chosen intentionally based on the FRR's
+// implementation. FRR interprets the upper 1 byte as an Administrative
+// Distance value and lower 3 bytes as an actual metric. In our case, AD is
+// 255. This is a special AD value which will never win the best path
+// selection. Please see FRR and Linux VRF's document for more details.
+//
+// FRR: https://docs.frrouting.org/en/latest/zebra.html#administrative-distance
+// Linux VRF: https://www.kernel.org/doc/Documentation/networking/vrf.txt
+func ensureUnreachableDefaultRoutes(n *NetConf, vrf *netlink.Vrf) error {
 	if n.hasV4() {
 		if err := netlink.RouteReplace(&netlink.Route{
 			Dst: &net.IPNet{
 				IP:   net.IPv4zero,
 				Mask: net.CIDRMask(0, 32),
 			},
-			Type:     unix.RTN_BLACKHOLE,
-			Priority: 100,
+			Type:     unix.RTN_UNREACHABLE,
+			Priority: 4278198272,
 			Table:    int(vrf.Table),
 		}); err != nil {
 			return err
@@ -231,8 +240,8 @@ func ensureBlackholeRoutes(n *NetConf, vrf *netlink.Vrf) error {
 				IP:   net.IPv6unspecified,
 				Mask: net.CIDRMask(0, 128),
 			},
-			Type:     unix.RTN_BLACKHOLE,
-			Priority: 100,
+			Type:     unix.RTN_UNREACHABLE,
+			Priority: 4278198272,
 			Table:    int(vrf.Table),
 		}); err != nil {
 			return err
@@ -254,9 +263,9 @@ func setupVRF(n *NetConf) (*netlink.Vrf, *current.Interface, error) {
 		return nil, nil, fmt.Errorf("failed to assign loopback addresses to VRF: %w", err)
 	}
 
-	// insert blackhole default route for the isolation
-	if err := ensureBlackholeRoutes(n, vrf); err != nil {
-		return nil, nil, fmt.Errorf("failed to insert blackhole default routes to VRF: %w", err)
+	// insert unreachable default route for the isolation
+	if err := ensureUnreachableDefaultRoutes(n, vrf); err != nil {
+		return nil, nil, fmt.Errorf("failed to insert unreachable default routes to VRF: %w", err)
 	}
 
 	return vrf, &current.Interface{

hostvrf_test.go

@@ -634,17 +634,17 @@ func TestCNIAddDel(t *testing.T) {
 					require.NoError(t, err)
 				})
 
-				t.Run("Blackhole routes are configured", func(t *testing.T) {
+				t.Run("Unreachable routes are configured", func(t *testing.T) {
 					err = c.ExecFunc(ctx, func(_ ns.NetNS) error {
 						if netConf.LoopbackAddressV4 != "" {
 							rt, ok := vrfRoutes["0.0.0.0/0"]
-							require.True(t, ok, "Blackhole default route for IPv4 is not configured")
-							require.Equal(t, 100, rt.Priority, "Metric for the blackhole default route for IPv4 should be 100")
+							require.True(t, ok, "Unreachable default route for IPv4 is not configured")
+							require.Equal(t, 4278198272, rt.Priority, "Metric for the unreachable default route for IPv4 must be 4278198272")
 						}
 						if netConf.LoopbackAddressV6 != "" {
 							rt, ok := vrfRoutes["::/0"]
-							require.True(t, ok, "Blackhole default route for IPv4 is not configured")
-							require.Equal(t, 100, rt.Priority, "Metric for the blackhole default route for IPv4 should be 100")
+							require.True(t, ok, "Unreachable default route for IPv6 is not configured")
+							require.Equal(t, 4278198272, rt.Priority, "Metric for the unreachable default route for IPv6 must be 4278198272")
 						}
 						return nil
 					})
@@ -805,7 +805,7 @@ func TestCNIAddDel(t *testing.T) {
 							)
 							require.NoError(t, err)
 							require.Len(t, routes, 2, "Unexpected number of IPv4 routes are left")
-							require.Contains(t, routes, "0.0.0.0/0", "Blackhole default route is missing after DEL")
+							require.Contains(t, routes, "0.0.0.0/0", "Unreachable default route is missing after DEL")
 							require.Contains(t, routes, netConf.LoopbackAddressV4+"/32", "Loopback route is missing after DEL")
 						}
 						if netConf.LoopbackAddressV6 != "" {
@@ -823,7 +823,7 @@ func TestCNIAddDel(t *testing.T) {
 							)
 							require.NoError(t, err)
 							require.Len(t, routes, 2, "Unexpected number of IPv6 routes are left")
-							require.Contains(t, routes, "::/0", "Blackhole default route is missing after DEL")
+							require.Contains(t, routes, "::/0", "Unreachable default route is missing after DEL")
 							require.Contains(t, routes, netConf.LoopbackAddressV6+"/128", "Loopback route is missing after DEL")
 						}
 						return nil