Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 35 additions & 72 deletions internal/schemas/manager/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ func (g *gitSource) Version(ctx context.Context) (string, error) {
return g.commitSHA, nil
}

func (g *gitSource) Resources(_ context.Context) (afero.Fs, error) {
func (g *gitSource) Resources(ctx context.Context) (afero.Fs, error) {
if g.fs != nil {
return g.fs, nil
}
Expand All @@ -217,6 +217,12 @@ func (g *gitSource) Resources(_ context.Context) (afero.Fs, error) {
var lastErr error

for attempt := 1; attempt <= maxCloneAttempts; attempt++ {
// When fetching several dependencies, another one failing cancels
// the group's context; don't keep retrying a doomed fetch.
if err := ctx.Err(); err != nil {
return nil, errors.Wrap(err, "clone cancelled")
Comment thread
haarchri marked this conversation as resolved.
Resolved
}

memFS = memfs.New()

headRef, err := g.cloner.CloneRepository(
Expand Down Expand Up @@ -432,12 +438,16 @@ func NewHTTPSource(dep v1alpha1.Dependency) Source {
}
}

// k8sOpenAPISource is a source that generates OpenAPI schemas for Kubernetes
// built-in APIs by downloading the swagger spec.
// k8sOpenAPISource is a source for the OpenAPI specs of Kubernetes built-in
// APIs. The specs are taken from a shallow checkout of kubernetes/kubernetes
// at the requested version, limited to api/openapi-spec, which carries the
// complete spec set: every API group and version, not just a fixed list. The
// checkout is an implementation detail of Resources: the source keeps its
// k8s:// identity and its configured version, so schema lock entries don't
// change and comparing against the lock needs no network access.
type k8sOpenAPISource struct {
k8sRef *v1alpha1.K8sDependency
client *http.Client
fs afero.Fs
git Source
}

func (k *k8sOpenAPISource) ID() string {
Expand All @@ -449,71 +459,7 @@ func (k *k8sOpenAPISource) Version(_ context.Context) (string, error) {
}

func (k *k8sOpenAPISource) Resources(ctx context.Context) (afero.Fs, error) {
if k.fs != nil {
return k.fs, nil
}

// Download the OpenAPI v3 spec from the Kubernetes repo
specURL := fmt.Sprintf("https://raw.githubusercontent.com/kubernetes/kubernetes/v%s/api/openapi-spec/v3/api__v1_openapi.json", strings.TrimPrefix(k.k8sRef.Version, "v"))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, specURL, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to create request")
}

resp, err := k.client.Do(req)
if err != nil {
return nil, errors.Wrap(err, "failed to fetch Kubernetes OpenAPI spec")
}
defer resp.Body.Close() //nolint:errcheck // nothing to do

if resp.StatusCode != http.StatusOK {
return nil, errors.Errorf("failed to download Kubernetes OpenAPI spec: HTTP %d", resp.StatusCode)
}

content, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPSize))
if err != nil {
return nil, errors.Wrap(err, "failed to read Kubernetes OpenAPI spec")
}

resultFS := afero.NewMemMapFs()
if err := afero.WriteFile(resultFS, "api__v1_openapi.json", content, 0o644); err != nil {
return nil, errors.Wrap(err, "failed to write OpenAPI spec")
}

// Also try to download the apis specs
apisGroups := []string{
"apis__apps__v1",
"apis__autoscaling__v1",
"apis__batch__v1",
"apis__networking.k8s.io__v1",
"apis__policy__v1",
"apis__rbac.authorization.k8s.io__v1",
"apis__storage.k8s.io__v1",
}

for _, group := range apisGroups {
groupURL := fmt.Sprintf("https://raw.githubusercontent.com/kubernetes/kubernetes/v%s/api/openapi-spec/v3/%s_openapi.json", strings.TrimPrefix(k.k8sRef.Version, "v"), group)
groupReq, err := http.NewRequestWithContext(ctx, http.MethodGet, groupURL, nil)
if err != nil {
continue
}

groupResp, err := k.client.Do(groupReq)
if err != nil {
continue
}

if groupResp.StatusCode == http.StatusOK {
groupContent, err := io.ReadAll(io.LimitReader(groupResp.Body, maxHTTPSize))
if err == nil {
_ = afero.WriteFile(resultFS, group+"_openapi.json", groupContent, 0o644)
}
}
_ = groupResp.Body.Close()
}

k.fs = resultFS
return k.fs, nil
return k.git.Resources(ctx)
}

func (k *k8sOpenAPISource) Type() SourceType {
Expand All @@ -524,8 +470,25 @@ func (k *k8sOpenAPISource) Type() SourceType {
func NewK8sSource(dep v1alpha1.Dependency) Source {
return &k8sOpenAPISource{
k8sRef: dep.K8s,
client: &http.Client{
Timeout: defaultHTTPTimeout,
git: NewGitSource(k8sAsGitDependency(dep), &git.DefaultCloner{}, &git.HTTPSAuthProvider{}),
}
}

// k8sAsGitDependency converts a k8s dependency to the git dependency its
// specs are fetched through. The dependency keeps its k8s type, so the git
// source treats the files as OpenAPI documents.
func k8sAsGitDependency(dep v1alpha1.Dependency) v1alpha1.Dependency {
version := dep.K8s.Version
if !strings.HasPrefix(version, "v") {
version = "v" + version
}
return v1alpha1.Dependency{
Type: v1alpha1.DependencyTypeK8s,
K8s: dep.K8s,
Git: &v1alpha1.GitDependency{
Repository: "https://github.com/kubernetes/kubernetes",
Ref: version,
Path: "api/openapi-spec",
},
}
}
106 changes: 106 additions & 0 deletions internal/schemas/manager/source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ limitations under the License.
package manager

import (
"context"
"errors"
"strings"
"testing"

"github.com/go-git/go-billy/v5"
Expand Down Expand Up @@ -680,3 +683,106 @@ func TestCalculateFilesystemHashDifferentContent(t *testing.T) {
})
}
}

// TestK8sSource pins the k8s source's contract: the git checkout backing
// Resources is an implementation detail, so the source's identity and
// version — what the schema lock records and what lock comparison reads —
// stay those of the k8s dependency and need no network access.
func TestK8sSource(t *testing.T) {
Comment thread
haarchri marked this conversation as resolved.
Resolved
t.Parallel()

dep := v1alpha1.Dependency{
Type: v1alpha1.DependencyTypeK8s,
K8s: &v1alpha1.K8sDependency{Version: "1.34.0"},
}

t.Run("LockIdentityWithoutNetwork", func(t *testing.T) {
t.Parallel()

// The cloner must stay untouched: Version runs before the lock
// comparison, so a clone here would fetch kubernetes/kubernetes
// even when the recorded revision already matches.
cloner := &mockCloner{}
src := &k8sOpenAPISource{
k8sRef: dep.K8s,
git: NewGitSource(k8sAsGitDependency(dep), cloner, &mockAuthProvider{}),
}

if got, want := src.ID(), "k8s://1.34.0"; got != want {
t.Errorf("ID() = %q, want %q", got, want)
}
version, err := src.Version(t.Context())
if err != nil {
t.Fatalf("Version() failed: %v", err)
}
if got, want := version, "1.34.0"; got != want {
t.Errorf("Version() = %q, want %q", got, want)
}
if got := src.Type(); got != SourceTypeOpenAPI {
t.Errorf("Type() = %q, want %q", got, SourceTypeOpenAPI)
}
if cloner.cloneCalls != 0 {
t.Errorf("ID/Version/Type cloned %d times, want no clones", cloner.cloneCalls)
}
})

t.Run("ResourcesDelegateToGit", func(t *testing.T) {
t.Parallel()

cloner := &mockCloner{ref: plumbing.NewHashReference("refs/tags/v1.34.0", plumbing.NewHash("abc123def456789012345678901234567890abcd"))}
src := &k8sOpenAPISource{
k8sRef: dep.K8s,
git: NewGitSource(k8sAsGitDependency(dep), cloner, &mockAuthProvider{}),
}

fs, err := src.Resources(t.Context())
if err != nil {
t.Fatalf("Resources() failed: %v", err)
}
if fs == nil {
t.Fatal("Resources() returned no filesystem")
}
if cloner.cloneCalls != 1 {
t.Errorf("Resources() cloned %d times, want 1", cloner.cloneCalls)
}
})

t.Run("GitDependencyShape", func(t *testing.T) {
t.Parallel()

gitDep := k8sAsGitDependency(dep)
if got, want := gitDep.Git.Repository, "https://github.com/kubernetes/kubernetes"; got != want {
t.Errorf("Repository = %q, want %q", got, want)
}
if got, want := gitDep.Git.Ref, "v1.34.0"; got != want {
t.Errorf("Ref = %q, want %q (the configured version always gains a v prefix)", got, want)
}
if got, want := gitDep.Git.Path, "api/openapi-spec"; got != want {
t.Errorf("Path = %q, want %q", got, want)
}
})
}

// TestGitSourceResourcesCancelled verifies a cancelled context stops the
// clone retry loop: when fetching several dependencies, another one failing
// cancels the group's context, and a doomed fetch must not keep retrying.
func TestGitSourceResourcesCancelled(t *testing.T) {
t.Parallel()

cloner := &mockCloner{cloneErr: errors.New("network down")}
src := NewGitSource(v1alpha1.Dependency{
Type: v1alpha1.DependencyTypeCRD,
Git: &v1alpha1.GitDependency{Repository: "https://example.org/repo", Ref: "main"},
}, cloner, &mockAuthProvider{})

ctx, cancel := context.WithCancel(t.Context())
cancel()

_, err := src.Resources(ctx)
if err == nil || !strings.Contains(err.Error(), "clone cancelled") {
t.Fatalf("Resources() with cancelled context: want clone cancelled error, got %v", err)
Comment thread
haarchri marked this conversation as resolved.
Resolved
}
if cloner.cloneCalls != 0 {
t.Errorf("Resources() cloned %d times after cancellation, want none", cloner.cloneCalls)
}
}