Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: ListNodeImageVersions + shared image gallery support #526

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1cb133c
feat: support shared image galleries inside of karpenter
Bryce-Soghigian Oct 15, 2024
db34e35
chore: populating image stubs for shared image galleries
Bryce-Soghigian Oct 15, 2024
ac5e943
fix: progress
Bryce-Soghigian Oct 15, 2024
78fc1ec
fix: PopulateResourceStub accessing the wrong index
Bryce-Soghigian Oct 15, 2024
24ec40f
test: properly testing ListNodeImageVersions
Bryce-Soghigian Oct 16, 2024
c79ecfd
refactor: rename symbol for SIG Subscription id
Bryce-Soghigian Oct 16, 2024
f19c719
test: conditional use of sig dependent on the managed karpenter flag
Bryce-Soghigian Oct 16, 2024
107b41f
refactor: removing panics used in testing
Bryce-Soghigian Oct 16, 2024
9c179cc
test: adding RBAC and helm values to the template for SIG Gallery logic
Bryce-Soghigian Oct 16, 2024
3f7f761
fix: bug in azure linux sig image resolution
Bryce-Soghigian Oct 16, 2024
0efac0d
chore: update cleanupenv to handle inflate too ratehr than just job pods
Bryce-Soghigian Oct 16, 2024
637e30e
test: fix randomized test order flake
Bryce-Soghigian Oct 16, 2024
0d7a98a
ci: shadow declaration
Bryce-Soghigian Oct 16, 2024
13b410d
refactor: comment wording
Bryce-Soghigian Oct 16, 2024
05e4ac9
test: validate all image ids are resolved correctly
Bryce-Soghigian Oct 16, 2024
283484d
fix: adding filtering for duplicate sku + os combinations and filteri…
Bryce-Soghigian Oct 16, 2024
227f7d6
refactor: renaming var
Bryce-Soghigian Oct 30, 2024
340f481
refactor: rename the managedKarpenter reference to UseSIG
Bryce-Soghigian Nov 14, 2024
1901a83
refactor: spelling
Bryce-Soghigian Nov 14, 2024
cc07eb7
fix: v1 migration for test
Bryce-Soghigian Nov 15, 2024
4b05f7f
ci: lint
Bryce-Soghigian Nov 15, 2024
29deb55
fix: lint
Bryce-Soghigian Nov 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Makefile-az.mk
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ else
AZURE_ACR_NAME ?= $(COMMON_NAME)
endif

AZURE_SIG_SUBSCRIPTION_ID ?= $(AZURE_SUBSCRIPTION_ID)
AZURE_CLUSTER_NAME ?= $(COMMON_NAME)
AZURE_RESOURCE_GROUP_MC = MC_$(AZURE_RESOURCE_GROUP)_$(AZURE_CLUSTER_NAME)_$(AZURE_LOCATION)

Expand Down Expand Up @@ -47,6 +48,7 @@ az-acrimport: ## Imports an image to an acr registry
az acr import --name $(AZURE_ACR_NAME) --source "mcr.microsoft.com/oss/kubernetes/pause:3.6" --image "pause:3.6"

az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses)
kubectl delete deployments -n default --all
kubectl delete pods -n default --all
kubectl delete nodeclaims --all
kubectl delete nodepools --all
Expand Down Expand Up @@ -136,6 +138,11 @@ az-perm: ## Create role assignments to let Karpenter manage VMs and Network
az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) --role "Network Contributor" # in some case we create vnet here
@echo Consider "make az-configure-values"!

az-perm-sig: ## Create role assignments when testing with SIG images
$(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv))
az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-Ubuntu/providers/Microsoft.Compute/galleries/AKSUbuntu
az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-AzureLinux/providers/Microsoft.Compute/galleries/AKSAzureLinux

az-perm-subnet-custom: az-perm ## Create role assignments to let Karpenter manage VMs and Network (custom VNet)
$(eval VNET_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"))
$(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv))
Expand Down
6 changes: 6 additions & 0 deletions karpenter-values-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ controller:
value: ""
- name: AZURE_NODE_RESOURCE_GROUP
value: ${AZURE_RESOURCE_GROUP_MC}

# managed karpenter settings
- name: USE_SIG
value: "false"
- name: SIG_SUBSCRIPTION_ID
value: ${SIG_SUBSCRIPTION_ID}
serviceAccount:
name: ${KARPENTER_SERVICE_ACCOUNT_NAME}
annotations:
Expand Down
18 changes: 7 additions & 11 deletions pkg/cloudprovider/drift.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,21 +145,17 @@ func (c *CloudProvider) isImageVersionDrifted(

if vm.Properties == nil ||
vm.Properties.StorageProfile == nil ||
vm.Properties.StorageProfile.ImageReference == nil ||
vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == nil ||
*vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == "" {
logger.Debug("not using a CommunityGalleryImageID for nodeClaim %s", nodeClaim.Name)
vm.Properties.StorageProfile.ImageReference == nil {
return "", nil
}
CIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID)
SIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.ID)
vmImageID := lo.Ternary(SIGID != "", SIGID, CIGID)

vmImageID := *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID
var imageStub imagefamily.DefaultImageOutput
imageStub.PopulateImageTraitsFromID(vmImageID)

publicGalleryURL, communityImageName, _, err := imagefamily.ParseCommunityImageIDInfo(vmImageID)
if err != nil {
return "", err
}

expectedImageID, err := c.imageProvider.GetImageID(ctx, communityImageName, publicGalleryURL)
expectedImageID, err := c.imageProvider.GetLatestImageID(ctx, imageStub)
if err != nil {
return "", err
}
Expand Down
13 changes: 13 additions & 0 deletions pkg/fake/communityimageversionsapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ package fake

import (
"context"
"fmt"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
"github.com/samber/lo"

"github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily"
)
Expand Down Expand Up @@ -51,6 +53,17 @@ func (c *CommunityGalleryImageVersionsAPI) NewListPager(_ string, _ string, _ st
return runtime.NewPager(pagingHandler)
}

func (c *CommunityGalleryImageVersionsAPI) Get(_ context.Context, location string, publicGalleryName string, galleryImageName string, galleryImageVersionName string, options *armcompute.CommunityGalleryImageVersionsClientGetOptions) (armcompute.CommunityGalleryImageVersionsClientGetResponse, error) {
// TODO: Add case where this get doesn't work or succeed
return armcompute.CommunityGalleryImageVersionsClientGetResponse{
CommunityGalleryImageVersion: armcompute.CommunityGalleryImageVersion{
Identifier: &armcompute.CommunityGalleryIdentifier{
UniqueID: lo.ToPtr(fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", publicGalleryName, galleryImageName, galleryImageVersionName)),
},
},
}, nil
}

func (c *CommunityGalleryImageVersionsAPI) Reset() {
if c == nil {
return
Expand Down
Loading
Loading