Skip to content

Commit 4f922e8

Browse files
comtalysttallaxesmatthchr
authored
feat: CSE-based bootstrapping with bootstrapping client mode (#527)
* chore: update devcontainer go version * chore: refresh toolcain * chore: additional processing on verify (and migration to kube-system) * chore: bump dependencies * chore: refresh Helm charts * chore: update golangci config * chore: remove feature gate for drift * chore: update pre-commit tooling * chore: update the shape of main * chore: update the alt operator * chore: update the API (move kubelet config to AKSNodeClass) * chore: migrate cloud provider to v1 API * chore: migrate operator to v1 API * chore: migrate controllers to v1 API * chore: add nodeclass status controller * chore: migrate providers to v1 API * chore: migrate test pkg to v1 API * chore: update utils * chore: update and migrate E2E tests to v1 API * feat: refresh and relink CRDs * fix: move code generation into subfolders to fix golangci-lint (typecheck detecting multiple main.go) * fix: enable most of govet in golangci * fix(linting): exclude alt operator logger * fix: add nodeclass termination controller * fix(lint): restore linting on verify * feat: add nodeclass hash controller * fix: register additional nodeclass and status controllers * fix(e2e): better selection of karpenter pod for logs * fix(e2e): fix utilization suite * chore(e2e): add events to dump-logs (and simplify) * chore: rename v1 to corev1 * fix: remove extra $ * fix(e2e): add cilium label and taint * fix(e2e): fix labels and disruption for deamonset test * feat: update kubelet configuration * fix: conflicting nodeclaim.garbagecollcation controller name * chore: restore webhooks in alt operator * Clean up commented out webhook code * chore: swagger spec for nodebootstrapping GET and autogenerated client * feat: bootstrapping client provision mode * chore: general code improvements * fix: required VnetCidrs wasn't populated * chore: read error response as text * fix: handle the issue of mismatched error format in response body/header * chore: more accurate overhead subtraction for instancetype * fix: correct VM memory passing * test: some unit tests and improvements * chore: linter changes and small improvements * feat: makefile command for swagger client generation * chore: remove unused parameters in readResponse() * fix: swagger binary not found * chore: small improvements * chore: added missing fullstops in other env descriptions * chore: rename bootstrapping types to scriptless and customscripts to align with AgentBaker * fix(test): fix test for credential provider URL in custom data * Make webhooks work in AKS CCP context (#537) This requires quite a bit of hacking, mostly overriding certain things in the ctx. The major items are: * Copy and modify knative/pkg/webhook/resourcesemantics/conversion to support CRD clientConfig.url in addition to clientConfig.service. * Copy and modify karpenter/pkg/webhooks/webhooks.go to support overriding the informer factory, so that we can point it at the CCP APIServer rather than overlay. * Override Start and supporting methods on the provider specific operator in pkg/operator/operator.go to allow invoking our modified version of karpenter/pkg/webhooks/webhooks.go. * chore: MaxPods is already defaulted * chore: remove failSwapOn from kubelet settings in AKSNodeClass * fix: populate nodeClaim.Status.ImageID * fix: record NodeClass hash and add drift on static fields * chore: rename variabled * fix: remove outdated comment * fix: typo * chore: update CRDs * feat: support custom kubeletconfig per v1 * chore: ignore false positive golint overflow check * test: fix option for provision mode in testing --------- Co-authored-by: tallaxes <[email protected]> Co-authored-by: Matthew Christopher <[email protected]>
1 parent 46fc0d3 commit 4f922e8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+4380
-68
lines changed

Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ coverage:
7575
go tool cover -html coverage.out -o coverage.html
7676

7777
verify: toolchain tidy download ## Verify code. Includes dependencies, linting, formatting, etc
78+
make az-swagger-generate-clients-raw
7879
go generate ./...
7980
hack/boilerplate.sh
8081
cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds

Makefile-az.mk

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,3 +358,11 @@ az-helm-install-snapshot: az-configure-values ## Install Karpenter snapshot rele
358358

359359
az-rmcrds: ## Delete Karpenter CRDs
360360
kubectl delete crd nodepools.karpenter.sh nodeclaims.karpenter.sh aksnodeclasses.karpenter.azure.com
361+
362+
az-swagger-generate-clients-raw:
363+
cd pkg/provisionclients && swagger generate client -f swagger/*.json
364+
hack/azure/temp_fix_get_bootstrapping_resp_error.sh
365+
366+
az-swagger-generate-clients: az-swagger-generate-clients-raw
367+
hack/boilerplate.sh
368+
make tidy

go.mod

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ require (
2121
github.com/blang/semver/v4 v4.0.0
2222
github.com/go-logr/logr v1.4.2
2323
github.com/go-logr/zapr v1.3.0
24+
github.com/go-openapi/errors v0.22.0
25+
github.com/go-openapi/runtime v0.28.0
26+
github.com/go-openapi/strfmt v0.23.0
27+
github.com/go-openapi/swag v0.23.0
28+
github.com/go-openapi/validate v0.24.0
2429
github.com/go-playground/validator/v10 v10.22.1
2530
github.com/google/go-cmp v0.6.0
2631
github.com/imdario/mergo v0.3.16
@@ -68,6 +73,7 @@ require (
6873
github.com/Azure/go-autorest/logger v0.2.1 // indirect
6974
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
7075
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
76+
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
7177
github.com/avast/retry-go v3.0.0+incompatible // indirect
7278
github.com/beorn7/perks v1.0.1 // indirect
7379
github.com/blendle/zapdriver v1.3.1 // indirect
@@ -83,9 +89,11 @@ require (
8389
github.com/go-kit/log v0.2.1 // indirect
8490
github.com/go-logfmt/logfmt v0.6.0 // indirect
8591
github.com/go-logr/stdr v1.2.2 // indirect
92+
github.com/go-openapi/analysis v0.23.0 // indirect
8693
github.com/go-openapi/jsonpointer v0.21.0 // indirect
8794
github.com/go-openapi/jsonreference v0.21.0 // indirect
88-
github.com/go-openapi/swag v0.23.0 // indirect
95+
github.com/go-openapi/loads v0.22.0 // indirect
96+
github.com/go-openapi/spec v0.21.0 // indirect
8997
github.com/go-playground/locales v0.14.1 // indirect
9098
github.com/go-playground/universal-translator v0.18.1 // indirect
9199
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
@@ -108,9 +116,12 @@ require (
108116
github.com/kylelemons/godebug v1.1.0 // indirect
109117
github.com/leodido/go-urn v1.4.0 // indirect
110118
github.com/mailru/easyjson v0.7.7 // indirect
119+
github.com/mitchellh/mapstructure v1.5.0 // indirect
111120
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
112121
github.com/modern-go/reflect2 v1.0.2 // indirect
113122
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
123+
github.com/oklog/ulid v1.3.1 // indirect
124+
github.com/opentracing/opentracing-go v1.2.0 // indirect
114125
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
115126
github.com/pkg/errors v0.9.1 // indirect
116127
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -122,6 +133,7 @@ require (
122133
github.com/shopspring/decimal v1.3.1 // indirect
123134
github.com/spf13/cobra v1.8.0 // indirect
124135
github.com/spf13/pflag v1.0.5 // indirect
136+
go.mongodb.org/mongo-driver v1.14.0 // indirect
125137
go.opencensus.io v0.24.0 // indirect
126138
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
127139
go.opentelemetry.io/otel v1.24.0 // indirect

go.sum

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
121121
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
122122
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
123123
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
124+
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
125+
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
124126
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
125127
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
126128
github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 h1:0dzaVod1XLEc38H4IB+KOgStoCt8RkCVI4t+XsSPrWE=
@@ -190,12 +192,26 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
190192
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
191193
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
192194
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
195+
github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU=
196+
github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo=
197+
github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w=
198+
github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE=
193199
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
194200
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
195201
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
196202
github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
203+
github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco=
204+
github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs=
205+
github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ=
206+
github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc=
207+
github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
208+
github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
209+
github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=
210+
github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=
197211
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
198212
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
213+
github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
214+
github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
199215
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
200216
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
201217
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -339,6 +355,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
339355
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
340356
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
341357
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
358+
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
359+
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
342360
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
343361
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
344362
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -350,10 +368,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
350368
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
351369
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
352370
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
371+
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
372+
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
353373
github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
354374
github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
355375
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
356376
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
377+
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
378+
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
357379
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
358380
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
359381
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
@@ -440,6 +462,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
440462
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
441463
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
442464
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
465+
go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
466+
go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
443467
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
444468
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
445469
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -454,6 +478,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
454478
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
455479
go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
456480
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
481+
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
482+
go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
457483
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
458484
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
459485
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
# Define the file path
4+
FILE="pkg/provisionclients/client/operations/node_bootstrapping_get_responses.go"
5+
6+
# Check if the file exists
7+
if [ ! -f "$FILE" ]; then
8+
echo "File $FILE does not exist."
9+
exit 1
10+
fi
11+
12+
# Use sed to delete the readResponse() method if it exists
13+
sed -i '/func (o \*NodeBootstrappingGetDefault) readResponse/,/^}/d' "$FILE"
14+
15+
echo "readResponse() method deleted from $FILE if it existed. This is for a temporary fix that is in node_bootstrapping_get_responses_override.go."

hack/toolchain.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ tools() {
2424
go install github.com/rhysd/actionlint/cmd/[email protected]
2525
go install github.com/mattn/[email protected]
2626
go install github.com/google/go-containerregistry/cmd/[email protected]
27+
go install github.com/go-swagger/go-swagger/cmd/[email protected]
2728

2829
if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then
2930
echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'."

pkg/consts/consts.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,7 @@ const (
2828
NetworkDataplaneAzure = "azure"
2929

3030
DefaultKubernetesMaxPods = 250
31+
32+
ProvisionModeAKSScriptless = "aksscriptless"
33+
ProvisionModeBootstrappingClient = "bootstrappingclient"
3134
)

pkg/operator/operator.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,12 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
117117
options.FromContext(ctx).ClusterEndpoint,
118118
azConfig.TenantID,
119119
azConfig.SubscriptionID,
120+
azConfig.ResourceGroup,
120121
azConfig.KubeletIdentityClientID,
121122
azConfig.NodeResourceGroup,
122123
azConfig.Location,
123124
vnetGUID,
125+
options.FromContext(ctx).ProvisionMode,
124126
)
125127
instanceTypeProvider := instancetype.NewDefaultProvider(
126128
azConfig.Location,
@@ -143,6 +145,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
143145
azConfig.Location,
144146
azConfig.NodeResourceGroup,
145147
azConfig.SubscriptionID,
148+
options.FromContext(ctx).ProvisionMode,
146149
)
147150

148151
return ctx, &Operator{

pkg/operator/options/options.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,9 @@ type Options struct {
7575
SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass
7676
setFlags map[string]bool
7777

78-
NodeResourceGroup string
78+
NodeResourceGroup string
79+
ProvisionMode string
80+
NodeBootstrappingServerURL string
7981
}
8082

8183
func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
@@ -85,12 +87,14 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
8587
fs.StringVar(&o.KubeletClientTLSBootstrapToken, "kubelet-bootstrap-token", env.WithDefaultString("KUBELET_BOOTSTRAP_TOKEN", ""), "[REQUIRED] The bootstrap token for new nodes to join the cluster.")
8688
fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.")
8789
fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", consts.NetworkPluginAzure), "The network plugin used by the cluster.")
88-
fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster")
90+
fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster.")
8991
fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.")
9092
fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.")
91-
fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR")
93+
fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR.")
9294
fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.")
93-
fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live")
95+
fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live.")
96+
fs.StringVar(&o.ProvisionMode, "provision-mode", env.WithDefaultString("PROVISION_MODE", consts.ProvisionModeAKSScriptless), "[UNSUPPORTED] The provision mode for the cluster.")
97+
fs.StringVar(&o.NodeBootstrappingServerURL, "nodebootstrapping-server-url", env.WithDefaultString("NODEBOOTSTRAPPING_SERVER_URL", ""), "[UNSUPPORTED] The url for the node bootstrapping provider server.")
9498
}
9599

96100
func (o Options) GetAPIServerName() string {

pkg/operator/options/options_validation.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ func (o Options) Validate() error {
3434
o.validateNetworkingOptions(),
3535
o.validateVMMemoryOverheadPercent(),
3636
o.validateVnetSubnetID(),
37+
o.validateProvisionMode(),
3738
validate.Struct(o),
3839
)
3940
}
@@ -83,6 +84,18 @@ func (o Options) validateVMMemoryOverheadPercent() error {
8384
return nil
8485
}
8586

87+
func (o Options) validateProvisionMode() error {
88+
if o.ProvisionMode != consts.ProvisionModeAKSScriptless && o.ProvisionMode != consts.ProvisionModeBootstrappingClient {
89+
return fmt.Errorf("provision-mode is invalid: %s", o.ProvisionMode)
90+
}
91+
if o.ProvisionMode == consts.ProvisionModeBootstrappingClient {
92+
if o.NodeBootstrappingServerURL == "" {
93+
return fmt.Errorf("nodebootstrapping-server-url is required when provision-mode is bootstrappingclient")
94+
}
95+
}
96+
return nil
97+
}
98+
8699
func (o Options) validateRequiredFields() error {
87100
if o.ClusterEndpoint == "" {
88101
return fmt.Errorf("missing field, cluster-endpoint")

0 commit comments

Comments
 (0)