Merge pull request kubernetes#130354 from siyuanfoundation/forward-api

KEP-4330: add forward compatibility for compatibility mode

Author: k8s-ci-robot

cmd/kube-apiserver/app/options/options_test.go

@@ -128,6 +128,8 @@ func TestAddFlags(t *testing.T) {
 		"--service-cluster-ip-range=192.168.128.0/17",
 		"--lease-reuse-duration-seconds=100",
 		"--emulated-version=test=1.31",
+		"--emulation-forward-compatible=true",
+		"--runtime-config-emulation-forward-compatible=true",
 	}
 	fs.Parse(args)
 	utilruntime.Must(componentGlobalsRegistry.Set())
@@ -136,17 +138,19 @@ func TestAddFlags(t *testing.T) {
 	expected := &ServerRunOptions{
 		Options: &controlplaneapiserver.Options{
 			GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
-				AdvertiseAddress:             netutils.ParseIPSloppy("192.168.10.10"),
-				CorsAllowedOriginList:        []string{"10.10.10.100", "10.10.10.200"},
-				MaxRequestsInFlight:          400,
-				MaxMutatingRequestsInFlight:  200,
-				RequestTimeout:               time.Duration(2) * time.Minute,
-				MinRequestTimeout:            1800,
-				StorageInitializationTimeout: time.Minute,
-				JSONPatchMaxCopyBytes:        int64(3 * 1024 * 1024),
-				MaxRequestBodyBytes:          int64(3 * 1024 * 1024),
-				ComponentGlobalsRegistry:     componentGlobalsRegistry,
-				ComponentName:                basecompatibility.DefaultKubeComponent,
+				AdvertiseAddress:                        netutils.ParseIPSloppy("192.168.10.10"),
+				CorsAllowedOriginList:                   []string{"10.10.10.100", "10.10.10.200"},
+				MaxRequestsInFlight:                     400,
+				MaxMutatingRequestsInFlight:             200,
+				RequestTimeout:                          time.Duration(2) * time.Minute,
+				MinRequestTimeout:                       1800,
+				StorageInitializationTimeout:            time.Minute,
+				JSONPatchMaxCopyBytes:                   int64(3 * 1024 * 1024),
+				MaxRequestBodyBytes:                     int64(3 * 1024 * 1024),
+				ComponentGlobalsRegistry:                componentGlobalsRegistry,
+				ComponentName:                           basecompatibility.DefaultKubeComponent,
+				EmulationForwardCompatible:              true,
+				RuntimeConfigEmulationForwardCompatible: true,
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{

pkg/controlplane/apiserver/apis.go

@@ -89,7 +89,12 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 	nonLegacy := []*genericapiserver.APIGroupInfo{}
 
 	// used later in the loop to filter the served resource by those that have expired.
-	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
+	resourceExpirationEvaluatorOpts := genericapiserver.ResourceExpirationEvaluatorOptions{
+		CurrentVersion:                          s.GenericAPIServer.EffectiveVersion.EmulationVersion(),
+		EmulationForwardCompatible:              s.GenericAPIServer.EmulationForwardCompatible,
+		RuntimeConfigEmulationForwardCompatible: s.GenericAPIServer.RuntimeConfigEmulationForwardCompatible,
+	}
+	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluatorFromOptions(resourceExpirationEvaluatorOpts)
 	if err != nil {
 		return err
 	}
@@ -107,10 +112,13 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 			continue
 		}
 
-		// Remove resources that serving kinds that are removed.
+		// Remove resources that serving kinds that are removed or not introduced yet at the current version.
 		// We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve.
 		// This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check.
-		resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap)
+		err = resourceExpirationEvaluator.RemoveUnavailableKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap, s.APIResourceConfigSource)
+		if err != nil {
+			return err
+		}
 		if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 {
 			klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName)
 			continue

staging/src/k8s.io/apimachinery/pkg/runtime/interfaces.go

@@ -259,6 +259,7 @@ type ObjectDefaulter interface {
 
 type ObjectVersioner interface {
 	ConvertToVersion(in Object, gv GroupVersioner) (out Object, err error)
+	PrioritizedVersionsForGroup(group string) []schema.GroupVersion
 }
 
 // ObjectConvertor converts an object to a different version.

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -154,6 +154,15 @@ type Config struct {
 	// EffectiveVersion determines which apis and features are available
 	// based on when the api/feature lifecyle.
 	EffectiveVersion basecompatibility.EffectiveVersion
+	// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
+	// have higher priority than APIs of the same group resource enabled at the emulation version.
+	// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
+	EmulationForwardCompatible bool
+	// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
+	// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
+	// If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version.
+	RuntimeConfigEmulationForwardCompatible bool
 	// FeatureGate is a way to plumb feature gate through if you have them.
 	FeatureGate featuregate.FeatureGate
 	// AuditBackend is where audit events are sent to.
@@ -839,8 +848,10 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		StorageReadinessHook:  NewStorageReadinessHook(c.StorageInitializationTimeout),
 		StorageVersionManager: c.StorageVersionManager,
 
-		EffectiveVersion: c.EffectiveVersion,
-		FeatureGate:      c.FeatureGate,
+		EffectiveVersion:                        c.EffectiveVersion,
+		EmulationForwardCompatible:              c.EmulationForwardCompatible,
+		RuntimeConfigEmulationForwardCompatible: c.RuntimeConfigEmulationForwardCompatible,
+		FeatureGate:                             c.FeatureGate,
 
 		muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
 	}

staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -19,6 +19,7 @@ package server
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -28,13 +29,18 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/registry/rest"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/klog/v2"
 )
 
+var alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
+
 // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
 type resourceExpirationEvaluator struct {
-	currentVersion *apimachineryversion.Version
-	isAlpha        bool
+	currentVersion                          *apimachineryversion.Version
+	emulationForwardCompatible              bool
+	runtimeConfigEmulationForwardCompatible bool
+	isAlpha                                 bool
 	// Special flag checking for the existence of alpha.0
 	// alpha.0 is a special case where everything merged to master is auto propagated to the release-1.n branch
 	isAlphaZero bool
@@ -50,20 +56,41 @@ type resourceExpirationEvaluator struct {
 
 // ResourceExpirationEvaluator indicates whether or not a resource should be served.
 type ResourceExpirationEvaluator interface {
-	// RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+	// RemoveUnavailableKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted or are introduced after the current version.
 	// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
-	RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage)
+	RemoveUnavailableKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error
 	// ShouldServeForVersion returns true if a particular version cut off is after the current version
 	ShouldServeForVersion(majorRemoved, minorRemoved int) bool
 }
 
+type ResourceExpirationEvaluatorOptions struct {
+	// CurrentVersion is the current version of the apiserver.
+	CurrentVersion *apimachineryversion.Version
+	// EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
+	// when resources of the same group and resource name but with lower priority are served.
+	EmulationForwardCompatible bool
+	// RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
+	// when the resource is explicitly enabled in runtime-config.
+	RuntimeConfigEmulationForwardCompatible bool
+}
+
 func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
+	opts := ResourceExpirationEvaluatorOptions{
+		CurrentVersion: currentVersion,
+	}
+	return NewResourceExpirationEvaluatorFromOptions(opts)
+}
+
+func NewResourceExpirationEvaluatorFromOptions(opts ResourceExpirationEvaluatorOptions) (ResourceExpirationEvaluator, error) {
+	currentVersion := opts.CurrentVersion
 	if currentVersion == nil {
 		return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
 	}
 	klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
 	ret := &resourceExpirationEvaluator{
-		strictRemovedHandlingInAlpha: false,
+		strictRemovedHandlingInAlpha:            false,
+		emulationForwardCompatible:              opts.EmulationForwardCompatible,
+		runtimeConfigEmulationForwardCompatible: opts.RuntimeConfigEmulationForwardCompatible,
 	}
 	// Only keeps the major and minor versions from input version.
 	ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
@@ -89,7 +116,8 @@ func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version)
 	return ret, nil
 }
 
-func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
+// isNotRemoved checks if a resource is removed due to the APILifecycleRemoved information.
+func (e *resourceExpirationEvaluator) isNotRemoved(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
 	internalPtr := resourceServingInfo.New()
 
 	target := gv
@@ -104,15 +132,6 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
 		return false
 	}
 
-	introduced, ok := versionedPtr.(introducedInterface)
-	if ok {
-		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
-		verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
-		if e.currentVersion.LessThan(verIntroduced) {
-			return false
-		}
-	}
-
 	removed, ok := versionedPtr.(removedInterface)
 	if !ok {
 		return true
@@ -155,13 +174,13 @@ type introducedInterface interface {
 
 // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
-func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
+func (e *resourceExpirationEvaluator) removeDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
 	versionsToRemove := sets.NewString()
 	for apiVersion := range sets.StringKeySet(versionedResourcesStorageMap) {
 		versionToResource := versionedResourcesStorageMap[apiVersion]
 		resourcesToRemove := sets.NewString()
 		for resourceName, resourceServingInfo := range versionToResource {
-			if !e.shouldServe(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
+			if !e.isNotRemoved(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
 				resourcesToRemove.Insert(resourceName)
 			}
 		}
@@ -189,6 +208,123 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
 	}
 }
 
+func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
+	e.removeDeletedKinds(groupName, versioner, versionedResourcesStorageMap)
+	return e.removeUnintroducedKinds(groupName, versioner, versionedResourcesStorageMap, apiResourceConfigSource)
+}
+
+// removeUnintroducedKinds inspects the storage map and modifies it in place by removing storage for kinds that are introduced after the current version.
+// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
+	versionsToRemove := sets.NewString()
+	prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
+	enabledResources := sets.NewString()
+
+	// iterate from the end to the front, so that we remove the lower priority versions first.
+	for i := len(prioritizedVersions) - 1; i >= 0; i-- {
+		apiVersion := prioritizedVersions[i].Version
+		versionToResource := versionedResourcesStorageMap[apiVersion]
+		if len(versionToResource) == 0 {
+			continue
+		}
+		resourcesToRemove := sets.NewString()
+		for resourceName, resourceServingInfo := range versionToResource {
+			// we check the resource enablement from low priority to high priority.
+			// If the same resource with a different version that we have checked so far is already enabled, that means some resource with the same resourceName and a lower priority version has been enabled.
+			// Then emulation forward compatibility for the version being checked now is made based on this information.
+			lowerPriorityEnabled := enabledResources.Has(resourceName)
+			shouldKeep, err := e.shouldServeBasedOnVersionIntroduced(schema.GroupVersionResource{Group: groupName, Version: apiVersion, Resource: resourceName},
+				versioner, resourceServingInfo, apiResourceConfigSource, lowerPriorityEnabled)
+			if err != nil {
+				return err
+			}
+			if !shouldKeep {
+				resourcesToRemove.Insert(resourceName)
+			} else if !alphaPattern.MatchString(apiVersion) {
+				// enabledResources is passed onto the next iteration to check the enablement of higher priority resources for emulation forward compatibility.
+				// But enablement alpha apis do not affect the enablement of other versions because emulation forward compatibility is not applicable to alpha apis.
+				enabledResources.Insert(resourceName)
+			}
+		}
+
+		for resourceName := range versionedResourcesStorageMap[apiVersion] {
+			if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
+				continue
+			}
+
+			klog.V(1).Infof("Removing resource %v.%v.%v because it is introduced after the current version %s per APILifecycle.", resourceName, apiVersion, groupName, e.currentVersion.String())
+			storage := versionToResource[resourceName]
+			storage.Destroy()
+			delete(versionToResource, resourceName)
+		}
+		versionedResourcesStorageMap[apiVersion] = versionToResource
+
+		if len(versionedResourcesStorageMap[apiVersion]) == 0 {
+			versionsToRemove.Insert(apiVersion)
+		}
+	}
+
+	for _, apiVersion := range versionsToRemove.List() {
+		gv := schema.GroupVersion{Group: groupName, Version: apiVersion}
+		if apiResourceConfigSource != nil && apiResourceConfigSource.VersionExplicitlyEnabled(gv) {
+			return fmt.Errorf(
+				"cannot enable version %s in runtime-config because all the resources have been introduced after the current version %s. Consider setting --runtime-config-emulation-forward-compatible=true",
+				gv, e.currentVersion)
+		}
+		klog.V(1).Infof("Removing version %v.%v because it is introduced after the current version %s and because it has no resources per APILifecycle.", apiVersion, groupName, e.currentVersion.String())
+		delete(versionedResourcesStorageMap, apiVersion)
+	}
+	return nil
+}
+
+func (e *resourceExpirationEvaluator) shouldServeBasedOnVersionIntroduced(gvr schema.GroupVersionResource, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage,
+	apiResourceConfigSource serverstorage.APIResourceConfigSource, lowerPriorityEnabled bool) (bool, error) {
+	verIntroduced := apimachineryversion.MajorMinor(0, 0)
+	internalPtr := resourceServingInfo.New()
+
+	target := gvr.GroupVersion()
+	// honor storage that overrides group version (used for things like scale subresources)
+	if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
+		target = versionProvider.GroupVersionKind(target).GroupVersion()
+	}
+
+	versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return false, err
+	}
+
+	introduced, ok := versionedPtr.(introducedInterface)
+	if ok {
+		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+		verIntroduced = apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+	}
+	// should serve resource introduced at or before the current version.
+	if e.currentVersion.AtLeast(verIntroduced) {
+		return true, nil
+	}
+	// the rest of the function is to determine if a resource introduced after current version should be served. (only applicable in emulation mode.)
+
+	// if a lower priority version of the resource has been enabled, the same resource with higher priority
+	// should also be enabled if emulationForwardCompatible = true.
+	if e.emulationForwardCompatible && lowerPriorityEnabled {
+		return true, nil
+	}
+	if apiResourceConfigSource == nil {
+		return false, nil
+	}
+	// could explicitly enable future resources in runtime-config forward compatible mode.
+	if e.runtimeConfigEmulationForwardCompatible && (apiResourceConfigSource.ResourceExplicitlyEnabled(gvr) || apiResourceConfigSource.VersionExplicitlyEnabled(gvr.GroupVersion())) {
+		return true, nil
+	}
+	// return error if a future resource is explicit enabled in runtime-config but runtimeConfigEmulationForwardCompatible is false.
+	if apiResourceConfigSource.ResourceExplicitlyEnabled(gvr) {
+		return false, fmt.Errorf("cannot enable resource %s in runtime-config because it is introduced at %s after the current version %s. Consider setting --runtime-config-emulation-forward-compatible=true",
+			gvr, verIntroduced, e.currentVersion)
+	}
+	return false, nil
+}
+
 func shouldRemoveResourceAndSubresources(resourcesToRemove sets.String, resourceName string) bool {
 	for _, resourceToRemove := range resourcesToRemove.List() {
 		if resourceName == resourceToRemove {