[release/8.0] Backport improvements to Http and Ssl stress tests (#122141)

Author: MihaZupan

eng/docker/build-docker-sdk.ps1

@@ -9,10 +9,11 @@ Param(
   [switch][Alias('w')]$buildWindowsContainers
 )
 
-$dotNetVersion="8.0"
 $ErrorActionPreference = "Stop"
 
 $REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
+[xml]$xml = Get-Content (Join-Path $REPO_ROOT_DIR "eng\Versions.props")
+$VERSION="$($xml.Project.PropertyGroup.MajorVersion[0]).$($xml.Project.PropertyGroup.MinorVersion[0])"
 
 $dockerFilePrefix="$PSScriptRoot/libraries-sdk"
 
@@ -29,7 +30,7 @@ if ($buildWindowsContainers)
   }
 
   $dockerFile="$dockerFilePrefix.windows.Dockerfile"
-  
+
   # Collect the following artifacts to folder, that will be used as build context for the container,
   # so projects can build and test against the live-built runtime:
   # 1. Reference assembly pack (microsoft.netcore.app.ref)
@@ -54,15 +55,16 @@ if ($buildWindowsContainers)
                      -Destination $dockerContext\targetingpacks.targets
   Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop `
                      -Destination $dockerContext\msquic-interop
-  
+
   # In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
-  $testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
+  $testHostAspNetCorePath="$dockerContext\testhost\net$VERSION-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
   if (Test-Path $testHostAspNetCorePath) {
     Remove-Item -Recurse -Force $testHostAspNetCorePath
   }
-  
+
   docker build --tag $imageName `
     --build-arg CONFIGURATION=$configuration `
+    --build-arg VERSION=$VERSION `
     --file $dockerFile `
     $dockerContext
 }
@@ -73,6 +75,7 @@ else
 
   docker build --tag $imageName `
       --build-arg CONFIGURATION=$configuration `
+      --build-arg "VERSION=$VERSION" `
       --file $dockerFile `
       $REPO_ROOT_DIR
 }

eng/docker/build-docker-sdk.sh

@@ -23,8 +23,12 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 imagename="dotnet-sdk-libs-current"
 configuration="Release"
+repo_root=$(git -C "$scriptroot" rev-parse --show-toplevel)
+major_version=$(grep -oP '(?<=<MajorVersion>).*?(?=</MajorVersion>)' "$repo_root/eng/Versions.props")
+minor_version=$(grep -oP '(?<=<MinorVersion>).*?(?=</MinorVersion>)' "$repo_root/eng/Versions.props")
+version="$major_version.$minor_version"
 
-while [[ $# > 0 ]]; do
+while [[ $# -gt 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -imagename|-t)
@@ -41,12 +45,12 @@ while [[ $# > 0 ]]; do
   esac
 done
 
-repo_root=$(git rev-parse --show-toplevel)
 docker_file="$scriptroot/libraries-sdk.linux.Dockerfile"
 
 docker build --tag $imagename \
     --build-arg CONFIGURATION=$configuration \
+    --build-arg VERSION=$version \
     --file $docker_file \
     $repo_root
 
-exit $?
+exit $?

eng/docker/libraries-sdk.linux.Dockerfile

@@ -1,6 +1,6 @@
 # Builds and copies library artifacts into target dotnet sdk image
 ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
-ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:7.0-bullseye-slim
+ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:latest
 
 FROM $BUILD_BASE_IMAGE as corefxbuild
 
@@ -12,12 +12,15 @@ RUN ./build.sh clr+libs -runtimeconfiguration Release -configuration $CONFIGURAT
 
 FROM $SDK_BASE_IMAGE as target
 
-ARG VERSION=8.0
+ARG VERSION
 ARG CONFIGURATION=Release
 ENV _DOTNET_INSTALL_CHANNEL=$VERSION
 
+# remove the existing SDK, we want to start from a clean slate with latest daily
+RUN rm -rf /usr/share/dotnet
+
 # Install latest daily SDK:
-RUN wget https://dot.net/v1/dotnet-install.sh
+RUN wget https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh
 RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily --install-dir /usr/share/dotnet
 
 # Collect the following artifacts under /live-runtime-artifacts,
@@ -48,8 +51,8 @@ COPY --from=corefxbuild \
     /repo/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop \
     /live-runtime-artifacts/msquic-interop
 
-# Add AspNetCore bits to testhost:
-ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+# Add AspNetCore bits to testhost, there should be only one version since we started from an image without existing SDK:
+ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/*"
 ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
 RUN mkdir -p $_ASPNETCORE_DEST
 RUN cp -r $_ASPNETCORE_SOURCE $_ASPNETCORE_DEST

eng/docker/libraries-sdk.windows.Dockerfile

@@ -1,6 +1,6 @@
 # escape=`
 # Simple Dockerfile which copies clr and library build artifacts into target dotnet sdk image
-ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:7.0-nanoserver-ltsc2022
+ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:8.0-nanoserver-ltsc2022
 FROM $SDK_BASE_IMAGE as target
 
 SHELL ["pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
@@ -11,15 +11,18 @@ ARG CONFIGURATION=Release
 
 USER ContainerAdministrator
 
-RUN Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
+# remove the existing ASP.NET SDK, we want to keep only the latest one we download later
+RUN Remove-Item -Force -Recurse 'C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/*'
+
+RUN Invoke-WebRequest -Uri https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
 RUN & .\dotnet-install.ps1 -Channel $env:_DOTNET_INSTALL_CHANNEL -Quality daily -InstallDir 'C:/Program Files/dotnet'
 
 USER ContainerUser
 
 COPY . /live-runtime-artifacts
 
 # Add AspNetCore bits to testhost:
-ENV _ASPNETCORE_SOURCE="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+ENV _ASPNETCORE_SOURCE="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/*"
 ENV _ASPNETCORE_DEST="C:/live-runtime-artifacts/testhost/net$VERSION-windows-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
 RUN & New-Item -ItemType Directory -Path $env:_ASPNETCORE_DEST
 RUN Copy-Item -Recurse -Path $env:_ASPNETCORE_SOURCE -Destination $env:_ASPNETCORE_DEST

eng/pipelines/libraries/stress/http.yml

@@ -12,6 +12,7 @@ schedules:
   branches:
     include:
     - main
+    - release/*.0
     - release/*-staging
 
 variables:
@@ -35,6 +36,7 @@ extends:
         timeoutInMinutes: 240
         variables:
           DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
+          DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
         pool:
           name: $(DncEngPublicBuildPool)
           demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
@@ -58,32 +60,29 @@ extends:
 
         - bash: |
             cd '$(httpStressProject)'
-            export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
-            export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
-            export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
-            export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
+            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
+            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
+            mkdir -p $DUMPS_SHARE
             docker-compose up --abort-on-container-exit --no-color
           timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
           displayName: Run HttpStress - HTTP 3.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
         - bash: |
             cd '$(httpStressProject)'
-            export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
-            export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
-            export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
-            export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
+            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
+            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
+            mkdir -p $DUMPS_SHARE
             docker-compose down
             docker-compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 2.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
         - bash: |
             cd '$(httpStressProject)'
-            export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
-            export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
-            export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
-            export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
+            export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
+            export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
+            mkdir -p $DUMPS_SHARE
             docker-compose down
             docker-compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 1.1
@@ -103,12 +102,7 @@ extends:
         timeoutInMinutes: 150
         variables:
           DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
-
-          # The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
-          # V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
-          # This is worked around by handpicking the V2 executable.
-          # The workaround should be removed when the official fix is propagated into 1es-windows-2022-open, or when we switch to another image.
-          DOCKER_COMPOSE_CMD: "C:/ProgramData/docker/cli-plugins/docker-compose.exe"
+          DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
         pool:
           name: $(DncEngPublicBuildPool)
           demands: ImageOverride -equals 1es-windows-2022-open
@@ -120,19 +114,12 @@ extends:
           lfs: false
 
         - powershell: |
-            # Workaround for https://github.com/microsoft/azure-pipelines-agent/issues/4554. Undo when the image bug is fixed.
-            Remove-Item -Path "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\Microsoft.VCToolsVersion.v143.default.txt"
-
             $(dockerfilesFolder)/build-docker-sdk.ps1 -w -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION)
             echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
           name: buildRuntime
           displayName: Build CLR and Libraries
 
         - powershell: |
-            $env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/"
-            $env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/"
-            New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
-            New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
             $(httpStressProject)/run-docker-compose.ps1 -w -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage)
             echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
           name: buildStress
@@ -147,37 +134,28 @@ extends:
 
         - powershell: |
             cd '$(httpStressProject)'
-            $env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
-            $env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
-            New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
-            New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
-            $env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
-            $env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
+            $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
+            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 3.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
         - powershell: |
             cd '$(httpStressProject)'
-            $env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
-            $env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
-            New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
-            New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
-            $env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
-            $env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
+            $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
+            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 2.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
         - powershell: |
             cd '$(httpStressProject)'
-            $env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
-            $env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
-            New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
-            New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
-            $env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
-            $env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
+            $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
+            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 1.1
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 

eng/pipelines/libraries/stress/ssl.yml

@@ -12,6 +12,7 @@ schedules:
   branches:
     include:
     - main
+    - release/*.0
     - release/*-staging
 
 variables:
@@ -34,6 +35,9 @@ extends:
       - job: linux
         displayName: Docker Linux
         timeoutInMinutes: 120
+        variables:
+          DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
+          DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
         pool:
           name: $(DncEngPublicBuildPool)
           demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
@@ -53,31 +57,32 @@ extends:
 
         - bash: |
             cd '$(sslStressProject)'
+            export STRESS_CLIENT_ARGS=$SSLSTRESS_CLIENT_ARGS
+            export STRESS_SERVER_ARGS=$SSLSTRESS_SERVER_ARGS
+            mkdir -p $DUMPS_SHARE
             docker-compose up --abort-on-container-exit --no-color
           displayName: Run SslStress
 
+        - publish: $(Build.ArtifactStagingDirectory)/dumps
+          artifact: DumpsLinux
+          condition: failed()
+
       - job: windows
         displayName: Docker NanoServer
         timeoutInMinutes: 120
         pool:
           name: $(DncEngPublicBuildPool)
           demands: ImageOverride -equals 1es-windows-2022-open
         variables:
-          # The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
-          # V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
-          # This is worked around by handpicking the V2 executable.
-          # The workaround should be removed when the official fix is propagated into 1es-windows-2022-open, or when we switch to another image.
-          DOCKER_COMPOSE_CMD: "C:/ProgramData/docker/cli-plugins/docker-compose.exe"
+          DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
+          DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
         steps:
         - checkout: self
           clean: true
           fetchDepth: 5
           lfs: false
 
         - powershell: |
-            # Workaround for https://github.com/microsoft/azure-pipelines-agent/issues/4554. Undo when the image bug is fixed.
-            Remove-Item -Path "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\Microsoft.VCToolsVersion.v143.default.txt"
-
             $(dockerfilesFolder)/build-docker-sdk.ps1 -w -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION)
           displayName: Build CLR and Libraries
 
@@ -87,5 +92,12 @@ extends:
 
         - powershell: |
             cd '$(sslStressProject)'
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            $env:STRESS_CLIENT_ARGS = $env:SSLSTRESS_CLIENT_ARGS
+            $env:STRESS_SERVER_ARGS = $env:SSLSTRESS_SERVER_ARGS
+            New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run SslStress
+
+        - publish: $(Build.ArtifactStagingDirectory)/dumps
+          artifact: DumpsWindows
+          condition: failed()