Merge branch 'main' into se-azurelinux

.github/workflows/AKSC_Deploy.yml

@@ -10,7 +10,7 @@ on:
       templateVersion:
         description: 'Template Version'
         required: false
-        default: '0.10.2'
+        default: '0.10.3'
         type: string
       rg:
         description: 'Resource Group name'

bicep/acragentpool.bicep

@@ -2,7 +2,7 @@ param location string = resourceGroup().location
 param acrName string
 param acrPoolSubnetId string = ''
 
-resource acr 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' existing =  {
+resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' existing =  {
   name: acrName
 }
 

bicep/aksagentpool.bicep

@@ -46,39 +46,50 @@ param osSKU string
 @description('Assign a public IP per node')
 param enableNodePublicIP bool = false
 
+@description('If the node pool should use VM spot instances')
+param spotInstance bool = false
+
 @description('Apply a default sku taint to Windows node pools')
 param autoTaintWindows bool = false
 
 var taints = autoTaintWindows ? union(nodeTaints, ['sku=Windows:NoSchedule']) : nodeTaints
 
-resource aks 'Microsoft.ContainerService/managedClusters@2023-07-02-preview' existing = {
+var spotProperties = {
+  scaleSetPriority: 'Spot'
+  scaleSetEvictionPolicy: 'Delete'
+  spotMaxPrice: -1
+}
+
+resource aks 'Microsoft.ContainerService/managedClusters@2023-08-02-preview' existing = {
   name: AksName
 }
 
-resource userNodepool 'Microsoft.ContainerService/managedClusters/agentPools@2021-10-01' = {
+resource userNodepool 'Microsoft.ContainerService/managedClusters/agentPools@2023-08-02-preview' = {
   parent: aks
   name: PoolName
-  properties: {
-    mode: 'User'
-    vmSize: agentVMSize
-    count: agentCount
-    minCount: autoScale ? agentCount : null
-    maxCount: autoScale ? agentCountMax : null
-    enableAutoScaling: autoScale
-    availabilityZones: !empty(availabilityZones) ? availabilityZones : null
-    osDiskType: osDiskType
-    osSKU: osSKU
-    osDiskSizeGB: osDiskSizeGB
-    osType: osType
-    maxPods: maxPods
-    type: 'VirtualMachineScaleSets'
-    vnetSubnetID: !empty(subnetId) ? subnetId : null
-    podSubnetID: !empty(podSubnetID) ? podSubnetID : null
-    upgradeSettings: {
-      maxSurge: '33%'
-    }
-    nodeTaints: taints
-    nodeLabels: nodeLabels
-    enableNodePublicIP: enableNodePublicIP
-  }
+  properties: union({
+      mode: 'User'
+      vmSize: agentVMSize
+      count: agentCount
+      minCount: autoScale ? agentCount : null
+      maxCount: autoScale ? agentCountMax : null
+      enableAutoScaling: autoScale
+      availabilityZones: !empty(availabilityZones) ? availabilityZones : null
+      osDiskType: osDiskType
+      osSKU: osSKU
+      osDiskSizeGB: osDiskSizeGB
+      osType: osType
+      maxPods: maxPods
+      type: 'VirtualMachineScaleSets'
+      vnetSubnetID: !empty(subnetId) ? subnetId : null
+      podSubnetID: !empty(podSubnetID) ? podSubnetID : null
+      upgradeSettings: spotInstance ? {} : {
+        maxSurge:  '33%' //Spot pools can't set max surge
+      }
+      nodeTaints: taints
+      nodeLabels: nodeLabels
+      enableNodePublicIP: enableNodePublicIP
+    },
+    spotInstance ? spotProperties : {}
+  )
 }

bicep/aksmetricalerts.bicep

@@ -720,7 +720,7 @@ resource PV_usage_violates_the_configured_threshold_for_clustername_CI_21 'micro
 }
 
 
-resource Daily_law_datacap 'Microsoft.Insights/scheduledQueryRules@2022-06-15' = {
+resource Daily_law_datacap 'Microsoft.Insights/scheduledQueryRules@2022-08-01-preview' = {
   name: 'Daily data cap breached for workspace ${logAnalyticsWorkspaceName} CIQ-1'
   location: logAnalyticsWorkspaceLocation
   properties: {

bicep/aksnetcontrib.bicep

@@ -17,7 +17,7 @@ var existingAksPodSubnetName = !empty(byoAKSPodSubnetId) ? split(byoAKSPodSubnet
 var existingAksSubnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[10] : ''
 var existingAksVnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[8] : ''
 
-resource existingvnet 'Microsoft.Network/virtualNetworks@2022-07-01' existing =  {
+resource existingvnet 'Microsoft.Network/virtualNetworks@2023-04-01' existing =  {
   name: existingAksVnetName
 }
 resource existingAksSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-07-01' existing = {

bicep/appgw.bicep

@@ -11,7 +11,7 @@ param appGWmaxCount int
 var appgwName = 'agw-${resourceName}'
 var appgwResourceId = resourceId('Microsoft.Network/applicationGateways', '${appgwName}')
 
-resource appgwpip 'Microsoft.Network/publicIPAddresses@2020-07-01' = {
+resource appgwpip 'Microsoft.Network/publicIPAddresses@2023-04-01' = {
   name: 'pip-agw-${resourceName}'
   location: location
   sku: {
@@ -131,7 +131,7 @@ var appGwZones = !empty(availabilityZones) ? availabilityZones : []
 
 // 'identity' is always set until this is fixed:
 // https://github.com/Azure/bicep/issues/387#issuecomment-885671296
-resource appgw 'Microsoft.Network/applicationGateways@2020-07-01' = if (!empty(userAssignedIdentity)) {
+resource appgw 'Microsoft.Network/applicationGateways@2023-04-01' = if (!empty(userAssignedIdentity)) {
   name: appgwName
   location: location
   zones: appGwZones
@@ -147,7 +147,7 @@ resource appgw 'Microsoft.Network/applicationGateways@2020-07-01' = if (!empty(u
 param agicPrincipleId string
 var contributor = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')
 // https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-template#new-service-principal
-resource appGwAGICContrib 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = {
+resource appGwAGICContrib 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
   scope: appgw
   name: guid(resourceGroup().id, appgwName, 'appgwcont')
   properties: {
@@ -158,7 +158,7 @@ resource appGwAGICContrib 'Microsoft.Authorization/roleAssignments@2020-04-01-pr
 }
 
 var reader = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')
-resource appGwAGICRGReader 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = {
+resource appGwAGICRGReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
   scope: resourceGroup()
   name: guid(resourceGroup().id, appgwName, 'rgread')
   properties: {

bicep/automationrunbook/aksRbac.bicep

@@ -1,7 +1,7 @@
 param principalId string
 param aksName string
 
-resource aks 'Microsoft.ContainerService/managedClusters@2023-03-02-preview' existing = {
+resource aks 'Microsoft.ContainerService/managedClusters@2023-05-02-preview' existing = {
   name: aksName
 }
 

bicep/bicepconfig.json

@@ -5,6 +5,12 @@
         "enabled": true,
         "verbose": false,
         "rules": {
+            "prefer-unquoted-property-names" : {
+                "level": "warning"
+            },
+            "use-parent-property" : {
+                "level": "warning"
+            },
             "use-recent-api-versions" : {
                 "level": "warning"
             },

bicep/firewall.bicep

@@ -25,7 +25,7 @@ var managementIpConfig = {
   }
 }
 
-resource fw_pip 'Microsoft.Network/publicIPAddresses@2022-07-01' = {
+resource fw_pip 'Microsoft.Network/publicIPAddresses@2023-04-01' = {
   name: firewallPublicIpName
   location: location
   sku: {
@@ -38,7 +38,7 @@ resource fw_pip 'Microsoft.Network/publicIPAddresses@2022-07-01' = {
   }
 }
 
-resource fwManagementIp_pip 'Microsoft.Network/publicIPAddresses@2022-07-01' = if(fwSku=='Basic') {
+resource fwManagementIp_pip 'Microsoft.Network/publicIPAddresses@2023-04-01' = if(fwSku=='Basic') {
   name: firewallManagementPublicIpName
   location: location
   sku: {
@@ -91,7 +91,7 @@ resource fwDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if
 param appDnsZoneName string = ''
 
 var fw_name = 'afw-${resourceName}'
-resource fw 'Microsoft.Network/azureFirewalls@2022-01-01' = {
+resource fw 'Microsoft.Network/azureFirewalls@2023-04-01' = {
   name: fw_name
   location: location
   zones: !empty(availabilityZones) ? availabilityZones : []
@@ -122,7 +122,7 @@ resource fw 'Microsoft.Network/azureFirewalls@2022-01-01' = {
   }
 }
 
-resource fwPolicy 'Microsoft.Network/firewallPolicies@2022-01-01' = {
+resource fwPolicy 'Microsoft.Network/firewallPolicies@2023-04-01' = {
   name: 'afwp-${resourceName}'
   location: location
   properties: {

bicep/main.bicep

@@ -410,7 +410,7 @@ param acrUntaggedRetentionPolicy int = 30
 
 var acrName = 'cr${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}'
 
-resource acr 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = if (!empty(registries_sku)) {
+resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' = if (!empty(registries_sku)) {
   name: acrName
   location: location
   sku: {
@@ -622,7 +622,7 @@ resource appGwIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01
 var appgwName = 'agw-${resourceName}'
 var appgwResourceId = deployAppGw ? resourceId('Microsoft.Network/applicationGateways', '${appgwName}') : ''
 
-resource appgwpip 'Microsoft.Network/publicIPAddresses@2022-07-01' = if (deployAppGw) {
+resource appgwpip 'Microsoft.Network/publicIPAddresses@2023-04-01' = if (deployAppGw) {
   name: 'pip-agw-${resourceName}'
   location: location
   sku: {
@@ -761,7 +761,7 @@ var appgwProperties = union({
 } : {})
 
 // 'identity' is always set until this is fixed: https://github.com/Azure/bicep/issues/387#issuecomment-885671296
-resource appgw 'Microsoft.Network/applicationGateways@2022-07-01' = if (deployAppGw) {
+resource appgw 'Microsoft.Network/applicationGateways@2023-04-01' = if (deployAppGw) {
   name: appgwName
   location: location
   zones: !empty(availabilityZones) ? availabilityZones : []
@@ -921,6 +921,9 @@ var autoScale = agentCountMax > agentCount
 @description('Name for user node pool')
 param nodePoolName string = 'npuser01'
 
+@description('Config the user node pool as a spot instance')
+param nodePoolSpot bool = false
+
 @description('Allocate pod ips dynamically')
 param cniDynamicIpAllocation bool = false
 
@@ -1397,6 +1400,7 @@ module userNodePool '../bicep/aksagentpool.bicep' = if (!JustUseSystemPool){
     enableNodePublicIP: enableNodePublicIP
     osDiskSizeGB: osDiskSizeGB
     availabilityZones: availabilityZones
+    spotInstance: nodePoolSpot
   }
 }
 
@@ -1555,7 +1559,7 @@ resource AksDiags 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' =
   }
 }
 
-resource sysLog 'Microsoft.Insights/dataCollectionRules@2022-06-01' = if (createLaw && omsagent && enableSysLog) {
+resource sysLog 'Microsoft.Insights/dataCollectionRules@2021-09-01-preview' = if (createLaw && omsagent && enableSysLog) {
   name: 'MSCI-${location}-${aks.name}'
   location: location
   kind: 'Linux'
@@ -1638,7 +1642,7 @@ resource sysLog 'Microsoft.Insights/dataCollectionRules@2022-06-01' = if (create
   }
 }
 
-resource association 'Microsoft.Insights/dataCollectionRuleAssociations@2022-06-01' = if (createLaw && omsagent && enableSysLog) {
+resource association 'Microsoft.Insights/dataCollectionRuleAssociations@2021-09-01-preview' = if (createLaw && omsagent && enableSysLog) {
   name: '${aks.name}-${aks_law.name}-association'
   scope: aks
   properties: {
@@ -1744,7 +1748,7 @@ output LogAnalyticsId string = (createLaw) ? aks_law.id : ''
 @description('Create an Event Grid System Topic for AKS events')
 param createEventGrid bool = false
 
-resource eventGrid 'Microsoft.EventGrid/systemTopics@2021-12-01' = if(createEventGrid) {
+resource eventGrid 'Microsoft.EventGrid/systemTopics@2023-06-01-preview' = if(createEventGrid) {
   name: 'evgt-${aks.name}'
   location: location
   identity: {
@@ -1791,7 +1795,8 @@ var telemetryId = '3c1e2fc6-1c4b-44f9-8694-25d00ae30a3a-${location}'
     |__|     |_______||_______||_______||__|  |__| |_______|    |__|     | _| `._____|   |__|        |_______/ |_______|| _|      |_______| \______/      |__|     |__|  |__| |_______||__| \__|     |__|     */
 
 //  Telemetry Deployment
-resource telemetrydeployment 'Microsoft.Resources/deployments@2022-09-01' = if (enableTelemetry) {
+#disable-next-line no-deployments-resources
+resource telemetrydeployment 'Microsoft.Resources/deployments@2023-07-01' = if (enableTelemetry) {
   name: telemetryId
   properties: {
     mode: 'Incremental'

bicep/network.bicep

@@ -120,7 +120,7 @@ var fwmgmt_subnet = {
 }
 
 var routeFwTableName = 'rt-afw-${resourceName}'
-resource vnet_udr 'Microsoft.Network/routeTables@2022-07-01' = if (azureFirewalls) {
+resource vnet_udr 'Microsoft.Network/routeTables@2023-04-01' = if (azureFirewalls) {
   name: routeFwTableName
   location: location
   properties: {
@@ -206,7 +206,7 @@ var subnets = union(
 output debugSubnets array = subnets
 
 var vnetName = 'vnet-${resourceName}'
-resource vnet 'Microsoft.Network/virtualNetworks@2022-07-01' = {
+resource vnet 'Microsoft.Network/virtualNetworks@2023-04-01' = {
   name: vnetName
   location: location
   properties: {
@@ -239,7 +239,7 @@ module aks_vnet_con 'networksubnetrbac.bicep' = if (!empty(aksPrincipleId)) {
 
 /*   --------------------------------------------------------------------------  Private Link for ACR      */
 var privateLinkAcrName = 'pl-acr-${resourceName}'
-resource privateLinkAcr 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAcrId)) {
+resource privateLinkAcr 'Microsoft.Network/privateEndpoints@2023-04-01' = if (!empty(privateLinkAcrId)) {
   name: privateLinkAcrName
   location: location
   properties: {
@@ -279,7 +279,7 @@ resource privateDnsAcrLink 'Microsoft.Network/privateDnsZones/virtualNetworkLink
   }
 }
 
-resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAcrId))  {
+resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-04-01' = if (!empty(privateLinkAcrId))  {
   parent: privateLinkAcr
   name: 'default'
   properties: {
@@ -297,7 +297,7 @@ resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZo
 
 /*   --------------------------------------------------------------------------  Private Link for KeyVault      */
 var privateLinkAkvName = 'pl-akv-${resourceName}'
-resource privateLinkAkv 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAkvId)) {
+resource privateLinkAkv 'Microsoft.Network/privateEndpoints@2023-04-01' = if (!empty(privateLinkAkvId)) {
   name: privateLinkAkvName
   location: location
   properties: {
@@ -337,7 +337,7 @@ resource privateDnsAkvLink 'Microsoft.Network/privateDnsZones/virtualNetworkLink
   }
 }
 
-resource privateDnsAkvZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAkvId))  {
+resource privateDnsAkvZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-04-01' = if (!empty(privateLinkAkvId))  {
   parent: privateLinkAkv
   name: 'default'
   properties: {
@@ -361,7 +361,7 @@ var publicIpAddressName = 'pip-${bastionHostName}'
 ])
 param bastionSku string = 'Standard'
 
-resource bastionPip 'Microsoft.Network/publicIPAddresses@2022-07-01' = if(bastion) {
+resource bastionPip 'Microsoft.Network/publicIPAddresses@2023-04-01' = if(bastion) {
   name: publicIpAddressName
   location: location
   sku: {
@@ -373,7 +373,7 @@ resource bastionPip 'Microsoft.Network/publicIPAddresses@2022-07-01' = if(bastio
   }
 }
 
-resource bastionHost 'Microsoft.Network/bastionHosts@2022-11-01' = if(bastion) {
+resource bastionHost 'Microsoft.Network/bastionHosts@2023-04-01' = if(bastion) {
   name: bastionHostName
   location: location
   sku: {
@@ -405,7 +405,7 @@ resource log 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = if
 param CreateNsgFlowLogs bool = false
 
 var flowLogStorageName = take(replace(toLower('stflow${resourceName}${uniqueString(resourceGroup().id, resourceName)}'),'-',''),24)
-resource flowLogStor 'Microsoft.Storage/storageAccounts@2021-08-01' = if(CreateNsgFlowLogs && networkSecurityGroups) {
+resource flowLogStor 'Microsoft.Storage/storageAccounts@2023-01-01' = if(CreateNsgFlowLogs && networkSecurityGroups) {
   name: flowLogStorageName
   kind: 'StorageV2'
   sku: {
@@ -505,7 +505,7 @@ module nsgPrivateLinks 'nsg.bicep' = if(privateLinks && networkSecurityGroups) {
   ]
 }
 
-resource natGwIp 'Microsoft.Network/publicIPAddresses@2021-08-01' =  [for i in range(0, natGatewayPublicIps): if(natGateway) {
+resource natGwIp 'Microsoft.Network/publicIPAddresses@2023-04-01' =  [for i in range(0, natGatewayPublicIps): if(natGateway) {
   name: 'pip-${natGwName}-${i+1}'
   location: location
   sku: {
@@ -521,7 +521,7 @@ output natGwIpArr array = [for i in range(0, natGatewayPublicIps): natGateway ?
 
 var natGwName = 'ng-${resourceName}'
 
-resource natGw 'Microsoft.Network/natGateways@2021-08-01' = if(natGateway) {
+resource natGw 'Microsoft.Network/natGateways@2023-04-01' = if(natGateway) {
   name: natGwName
   location: location
   sku: {

bicep/networkwatcherflowlog.bicep

@@ -12,7 +12,7 @@ param workspaceId string = ''
 param workspaceResourceId string = ''
 param workspaceRegion string = resourceGroup().location
 
-resource networkWatcher 'Microsoft.Network/networkWatchers@2022-01-01' = {
+resource networkWatcher 'Microsoft.Network/networkWatchers@2023-04-01' = {
   name: 'NetworkWatcher_${location}'
   location: location
   properties: {}