diff --git a/.github/workflows/codeQL.yml b/.github/workflows/codeQL.yml new file mode 100644 index 000000000..6cb68940e --- /dev/null +++ b/.github/workflows/codeQL.yml @@ -0,0 +1,79 @@ +# This workflow generates weekly CodeQL reports for this repo, a security requirements. +# The workflow is adapted from the following reference: https://github.com/Azure-Samples/azure-functions-python-stream-openai/pull/2/files +# Generic comments on how to modify these file are left intactfor future maintenance. + +name: "CodeQL" + +on: + push: + branches: [ "main", "*" ] # TODO: remove development branch after approval + pull_request: + branches: [ "main", "*"] # TODO: remove development branch after approval + schedule: + - cron: '0 0 * * 1' # Weekly Monday run, needed for weekly reports + workflow_call: # allows to be invoked as part of a larger workflow + workflow_dispatch: # allows for the workflow to run manually see: https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow + +env: + solution: WebJobs.Extensions.DurableTask.sln + config: Release + +jobs: + + analyze: + name: Analyze + runs-on: windows-latest + permissions: + actions: read + contents: read + security-events: write + + + strategy: + fail-fast: false + matrix: + language: ['csharp'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + # Run CodeQL analysis + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" \ No newline at end of file diff --git a/.github/workflows/smoketest-dotnet-isolated-v4.yml b/.github/workflows/smoketest-dotnet-isolated-v4.yml index f818ff7ae..474f48448 100644 --- a/.github/workflows/smoketest-dotnet-isolated-v4.yml +++ b/.github/workflows/smoketest-dotnet-isolated-v4.yml @@ -19,7 +19,79 @@ jobs: steps: - uses: actions/checkout@v2 - # Validation is blocked on https://github.com/Azure/azure-functions-host/issues/7995 - - name: Run V4 .NET Isolated Smoke Test - run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/DotNetIsolated/Dockerfile -HttpStartPath api/StartHelloCitiesTyped -NoValidation + # Install .NET versions + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 6.x + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.x' + + - name: Set up .NET Core 8.x + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.x' + + # Install Azurite + - name: Set up Node.js (needed for Azurite) + uses: actions/setup-node@v3 + with: + node-version: '18.x' # Azurite requires at least Node 18 + + - name: Install Azurite + run: npm install -g azurite + + - name: Restore WebJobs extension + run: dotnet restore $solution + + - name: Build and pack WebJobs extension + run: cd ./src/WebJobs.Extensions.DurableTask && + mkdir ./out && + dotnet build -c Release WebJobs.Extensions.DurableTask.csproj --output ./out && + mkdir ~/packages && + dotnet nuget push ./out/Microsoft.Azure.WebJobs.Extensions.DurableTask.*.nupkg --source ~/packages && + dotnet nuget add source ~/packages + + - name: Build .NET Isolated Smoke Test + run: cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && + dotnet restore --verbosity normal && + dotnet build -c Release + + - name: Install core tools + run: npm i -g azure-functions-core-tools@4 --unsafe-perm true + + # Run smoke tests + # Unlike other smoke tests, the .NET isolated smoke tests run outside of a docker container, but to race conditions + # when building the smoke test app in docker, causing the build to fail. This is a temporary workaround until the + # root cause is identified and fixed. + + - name: Run smoke tests (Hello Cities) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/StartHelloCitiesTyped + + - name: Run smoke tests (Process Exit) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartProcessExitOrchestrator + + - name: Run smoke tests (Timeout) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartTimeoutOrchestrator + + - name: Run smoke tests (OOM) shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartOOMOrchestrator \ No newline at end of file diff --git a/.github/workflows/validate-build-analyzer.yml b/.github/workflows/validate-build-analyzer.yml new file mode 100644 index 000000000..4eb275c4e --- /dev/null +++ b/.github/workflows/validate-build-analyzer.yml @@ -0,0 +1,60 @@ +name: Validate Build (analyzer) + +on: + push: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + +env: + solution: WebJobs.Extensions.DurableTask.sln + config: Release + AzureWebJobsStorage: UseDevelopmentStorage=true + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution + + # Install Azurite + - name: Set up Node.js (needed for Azurite) + uses: actions/setup-node@v3 + with: + node-version: '18.x' # Azurite requires at least Node 18 + + - name: Install Azurite + run: npm install -g azurite + + # Run tests + - name: Run Analyzer tests + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/WebJobs.Extensions.DurableTask.Analyzers.Test/WebJobs.Extensions.DurableTask.Analyzers.Test.csproj + diff --git a/.github/workflows/validate-build-e2e.yml b/.github/workflows/validate-build-e2e.yml new file mode 100644 index 000000000..8056b73f8 --- /dev/null +++ b/.github/workflows/validate-build-e2e.yml @@ -0,0 +1,62 @@ +name: Validate Build (E2E tests) + +on: + push: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + +env: + solution: WebJobs.Extensions.DurableTask.sln + config: Release + AzureWebJobsStorage: UseDevelopmentStorage=true + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution + + # Install Azurite + - name: Set up Node.js (needed for Azurite) + uses: actions/setup-node@v3 + with: + node-version: '18.x' # Azurite requires at least Node 18 + + - name: Install Azurite + run: npm install -g azurite + + # Run tests + - name: Run FunctionsV2 tests (only DurableEntity_CleanEntityStorage test, which is flaky) + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj --filter "FullyQualifiedName~Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableTaskEndToEndTests.DurableEntity_CleanEntityStorage" + + - name: Run FunctionsV2 tests (all other E2E tests) + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj --filter "FullyQualifiedName~Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableTaskEndToEndTests&FullyQualifiedName!~Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableTaskEndToEndTests.DurableEntity_CleanEntityStorage" \ No newline at end of file diff --git a/.github/workflows/validate-build.yml b/.github/workflows/validate-build.yml new file mode 100644 index 000000000..9f740e834 --- /dev/null +++ b/.github/workflows/validate-build.yml @@ -0,0 +1,63 @@ +name: Validate Build (except E2E tests) + +on: + push: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + pull_request: + branches: + - main + - dev + paths-ignore: [ '**.md' ] + +env: + solution: WebJobs.Extensions.DurableTask.sln + config: Release + AzureWebJobsStorage: UseDevelopmentStorage=true + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution + + # Install Azurite + - name: Set up Node.js (needed for Azurite) + uses: actions/setup-node@v3 + with: + node-version: '18.x' # Azurite requires at least Node 18 + + - name: Install Azurite + run: npm install -g azurite + + # Run tests + - name: Run FunctionsV2 tests (except E2E tests) + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj --filter "FullyQualifiedName!~Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableTaskEndToEndTests" + + - name: Run Worker Extension tests + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test ./test/Worker.Extensions.DurableTask.Tests/Worker.Extensions.DurableTask.Tests.csproj + diff --git a/Directory.Build.targets b/Directory.Build.targets new file mode 100644 index 000000000..47c2b86a2 --- /dev/null +++ b/Directory.Build.targets @@ -0,0 +1,37 @@ + + + + + + + + + false + <_TranslateUrlPattern>(https://azfunc%40dev\.azure\.com/azfunc/internal/_git|https://dev\.azure\.com/azfunc/internal/_git|https://azfunc\.visualstudio\.com/internal/_git|azfunc%40vs-ssh\.visualstudio\.com:v3/azfunc/internal|git%40ssh\.dev\.azure\.com:v3/azfunc/internal)/([^/\.]+)\.(.+) + <_TranslateUrlReplacement>https://github.com/$2/$3 + + + + + + $([System.Text.RegularExpressions.Regex]::Replace($(ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement))) + + + + $([System.Text.RegularExpressions.Regex]::Replace(%(SourceRoot.ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement))) + + + + + \ No newline at end of file diff --git a/WebJobs.Extensions.DurableTask.sln b/WebJobs.Extensions.DurableTask.sln index 353e83805..b710584c2 100644 --- a/WebJobs.Extensions.DurableTask.sln +++ b/WebJobs.Extensions.DurableTask.sln @@ -18,6 +18,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .editorconfig = .editorconfig azure-pipelines-release-dotnet-isolated.yml = azure-pipelines-release-dotnet-isolated.yml azure-pipelines-release.yml = azure-pipelines-release.yml + Directory.Build.targets = Directory.Build.targets nuget.config = nuget.config README.md = README.md release_notes.md = release_notes.md @@ -94,7 +95,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerfTests", "PerfTests", "{ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DFPerfScenariosV4", "test\DFPerfScenarios\DFPerfScenariosV4.csproj", "{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml new file mode 100644 index 000000000..40f2d2a0f --- /dev/null +++ b/eng/ci/code-mirror.yml @@ -0,0 +1,21 @@ +trigger: + branches: + include: + # These are the branches we'll mirror to our internal ADO instance + # Keep this set limited as appropriate (don't mirror individual user branches). + - main + - dev + - v3.x + +resources: + repositories: + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: ci/variables/cfs.yml@eng + +extends: + template: ci/code-mirror.yml@eng diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml new file mode 100644 index 000000000..858ca3518 --- /dev/null +++ b/eng/ci/official-build.yml @@ -0,0 +1,50 @@ +variables: + - template: ci/variables/cfs.yml@eng + +trigger: + batch: true + branches: + include: + - main + - dev + - v3.x + +# CI only, does not trigger on PRs. +pr: none + +schedules: +# Build nightly to catch any new CVEs and report SDL often. +# We are also required to generated CodeQL reports weekly, so this +# helps us meet that. +- cron: "0 0 * * *" + displayName: Nightly Build + branches: + include: + - main + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: BuildAndSign + dependsOn: [] + jobs: + - template: /eng/templates/build.yml@self diff --git a/eng/ci/publish.yml b/eng/ci/publish.yml new file mode 100644 index 000000000..8b65519fa --- /dev/null +++ b/eng/ci/publish.yml @@ -0,0 +1,99 @@ +# This is our package-publishing pipeline. +# When executed, it automatically publishes the output of the 'official pipeline' (the nupkgs) to our internal ADO feed. +# It may optionally also publish the packages to NuGet, but that is gated behind a manual approval. + +trigger: none # only trigger is manual +pr: none # only trigger is manual + +# We include to this variable group to be able to access the NuGet API key +variables: +- group: durabletask_config + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + + pipelines: + - pipeline: officialPipeline # Reference to the pipeline to be used as an artifact source + source: 'durable-extension.official' + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: release + jobs: + + # ADO release + - job: adoRelease + displayName: ADO Release + templateContext: + inputs: + - input: pipelineArtifact + pipeline: officialPipeline # Pipeline reference, as defined in the resources section + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + # The preferred method of release on 1ES is by populating the 'output' section of a 1ES template. + # We use this method to release to ADO, but not to release to NuGet; this is explained in the 'nugetRelease' job. + # To read more about the 'output syntax', see: + # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs + # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages + outputs: + - output: nuget # 'nuget' is an output "type" for pushing to NuGet + displayName: 'Push to durabletask ADO feed' + packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling + packagesToPush: '$(System.DefaultWorkingDirectory)/**/*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg' + publishVstsFeed: '3f99e810-c336-441f-8892-84983093ad7f/c895696b-ce37-4fe7-b7ce-74333a04f8bf' + allowPackageConflicts: true + + # NuGet approval gate + - job: nugetApproval + displayName: NuGetApproval + pool: server # This task only works when executed on serverl pools, so this needs to be specified + steps: + # Wait for manual approval. + - task: ManualValidation@1 + inputs: + instructions: Confirm you want to push to NuGet + onTimeout: 'reject' + + # NuGet release + - job: nugetRelease + displayName: NuGet Release + dependsOn: + - nugetApproval + - adoRelease + condition: succeeded('nugetApproval', 'adoRelease') + templateContext: + inputs: + - input: pipelineArtifact + pipeline: officialPipeline # Pipeline reference as defined in the resources section + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + # Ideally, we would push to NuGet using the 1ES "template output" syntax, like we do for ADO. + # Unfortunately, that syntax does not allow for skipping duplicates when pushing to NuGet feeds + # (i.e; not failing the job when trying to push a package version that already exists on NuGet). + # This is a problem for us because our pipelines often produce multiple packages, and we want to be able to + # perform a 'nuget push *.nupkg' that skips packages already on NuGet while pushing the rest. + # Therefore, we use a regular .NET Core ADO Task to publish the packages until that usability gap is addressed. + steps: + - task: DotNetCoreCLI@2 + displayName: 'Push to nuget.org' + inputs: + command: custom + custom: nuget + arguments: 'push "*.nupkg" --api-key $(nuget_api_key) --skip-duplicate --source https://api.nuget.org/v3/index.json' + workingDirectory: '$(System.DefaultWorkingDirectory)/drop' \ No newline at end of file diff --git a/eng/templates/build.yml b/eng/templates/build.yml new file mode 100644 index 000000000..c68e0d44d --- /dev/null +++ b/eng/templates/build.yml @@ -0,0 +1,137 @@ +jobs: + - job: Build + + templateContext: + outputs: + - output: pipelineArtifact + path: $(build.artifactStagingDirectory) + artifact: drop + sbomBuildDropPath: $(build.artifactStagingDirectory) + sbomPackageName: 'Durable Functions Extension SBOM' + + steps: + + # Configure all the .NET SDK versions we need + - task: UseDotNet@2 + displayName: 'Use the .NET Core 2.1 SDK (required for build signing)' + inputs: + packageType: 'sdk' + version: '2.1.x' + + - task: UseDotNet@2 + displayName: 'Use the .NET Core 3.1 SDK' + inputs: + packageType: 'sdk' + version: '3.1.x' + + - task: UseDotNet@2 + displayName: 'Use the .NET 6 SDK' + inputs: + packageType: 'sdk' + version: '6.0.x' + + # Start by restoring all the dependencies. + - task: DotNetCoreCLI@2 + displayName: 'dotnet restore' + inputs: + command: restore + projects: '**/**/*.csproj' + feedsToUse: config + nugetConfigPath: 'nuget.config' + + # Build durable-extension + - task: VSBuild@1 + displayName: 'Build Durable Extension' + inputs: + solution: '**/WebJobs.Extensions.DurableTask.sln' + vsVersion: "16.0" + configuration: Release + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true # these flags make package build deterministic + + - template: ci/sign-files.yml@eng + parameters: + displayName: Sign assemblies + folderPath: 'src/WebJobs.Extensions.DurableTask/bin/Release' + pattern: '*DurableTask.dll' + signType: dll + + - template: ci/sign-files.yml@eng + parameters: + displayName: Sign assemblies + folderPath: 'src/Worker.Extensions.DurableTask/bin/Release' + pattern: '*DurableTask.dll' + signType: dll + + # dotnet pack + # Packaging needs to be a separate step from build. + # This will automatically pick up the signed DLLs. + - task: DotNetCoreCLI@2 + displayName: 'dotnet pack WebJobs.Extensions.DurableTask.csproj' + inputs: + command: pack + packagesToPack: 'src/**/WebJobs.Extensions.DurableTask.csproj' + configuration: Release + packDirectory: $(build.artifactStagingDirectory) + nobuild: true + + + # dotnet pack + # Packaging needs to be a separate step from build. + # This will automatically pick up the signed DLLs. + - task: DotNetCoreCLI@2 + displayName: 'dotnet pack Worker.Extensions.DurableTask.csproj' + inputs: + command: pack + packagesToPack: 'src/**/Worker.Extensions.DurableTask.csproj' + configuration: Release + packDirectory: $(build.artifactStagingDirectory) + nobuild: true + + # Remove redundant symbol package(s) + - script: | + echo *** Searching for .symbols.nupkg files to delete... + dir /s /b *.symbols.nupkg + + echo *** Deleting .symbols.nupkg files... + del /S /Q *.symbols.nupkg + + echo *** Listing remaining packages + dir /s /b *.nupkg + displayName: 'Remove Redundant Symbols Package(s)' + continueOnError: true + + - template: ci/sign-files.yml@eng + parameters: + displayName: Sign NugetPackages + folderPath: $(build.artifactStagingDirectory) + pattern: '*.nupkg' + signType: nuget + + # zip .NET in-proc perf tests + - task: DotNetCoreCLI@2 + displayName: 'Zip .NET in-proc perf tests' + inputs: + command: 'publish' + publishWebProjects: false + projects: '$(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/**/*.csproj' + arguments: '-o $(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/Output' + zipAfterPublish: true + modifyOutputPath: true + + # Move zip'ed .NET in-proc perf tests to the ADO publishing directory + - task: CopyFiles@2 + inputs: + SourceFolder: '$(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/Output/' + Contents: '**' + TargetFolder: '$(System.DefaultWorkingDirectory)/azure-functions-durable-extension/' + + # We also need to build the Java smoke test, for CodeQL compliance + # We don't need to build the other smoke tests, because they can be analyzed without being compiled, + # as they're interpreted languages. + # This could be a separate pipeline, but the task is so small that it's paired with the .NET code build + # for convenience. + - pwsh: | + cd ./test/SmokeTests/OOProcSmokeTests/durableJava/ + gradle build + ls + displayName: 'Build Java OOProc test (for CodeQL compliance)' \ No newline at end of file diff --git a/nuget.config b/nuget.config index 652118ea6..d580aab15 100644 --- a/nuget.config +++ b/nuget.config @@ -2,6 +2,7 @@ + diff --git a/release_notes.md b/release_notes.md index ad44bb9b3..3babbb06b 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,18 +1,22 @@ # Release Notes -## Microsoft.Azure.Functions.Worker.Extensions.DurableTask 1.2.1 +## Microsoft.Azure.Functions.Worker.Extensions.DurableTask (version) ### New Features -- Fix regression on `TerminateInstanceAsync` API causing invocations to fail with "unimplemented" exceptions (https://github.com/Azure/azure-functions-durable-extension/pull/2829). +- Fail fast if extendedSessionsEnabled set to 'true' for the worker type that doesn't support extended sessions (https://github.com/Azure/azure-functions-durable-extension/pull/2732). +- Added an `IFunctionsWorkerApplicationBuilder.ConfigureDurableExtension()` extension method for cases where auto-registration does not work (no source gen running). (#2950) ### Bug Fixes +- Fix custom connection name not working when using IDurableClientFactory.CreateClient() - contributed by [@hctan](https://github.com/hctan) +- Made durable extension for isolated worker configuration idempotent, allowing multiple calls safely. (#2950) + ### Breaking Changes ### Dependency Updates -## Microsoft.Azure.WebJobs.Extensions.DurableTask +## Microsoft.Azure.WebJobs.Extensions.DurableTask 2.13.7 ### New Features @@ -21,3 +25,5 @@ ### Breaking Changes ### Dependency Updates + +- Microsoft.DurableTask.Grpc to 1.3.0 diff --git a/samples/durable-client-managed-identity/aspnetcore-app/Controllers/TodoController.cs b/samples/durable-client-managed-identity/aspnetcore-app/Controllers/TodoController.cs new file mode 100644 index 000000000..c1c4ba6d8 --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/Controllers/TodoController.cs @@ -0,0 +1,130 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.ContextImplementations; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using TodoApi.Models; + +namespace TodoApi.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class TodoController : Controller + { + private readonly TodoContext _context; + private readonly IDurableClient _client; + + public TodoController(TodoContext context, IDurableClientFactory clientFactory, IConfiguration configuration) + { + _context = context; + + if (_context.TodoItems.Count() == 0) + { + _context.TodoItems.Add(new TodoItem { Name = "Item1" }); + _context.SaveChanges(); + } + + _client = clientFactory.CreateClient(new DurableClientOptions + { + ConnectionName = configuration["MyStorage"], + TaskHub = configuration["TaskHub"] + }); + } + + // GET: api/Todo + [HttpGet] + public async Task>> GetTodoItem() + { + return await _context.TodoItems.ToListAsync(); + } + + // GET: api/Todo/5 + [HttpGet("{id}")] + public async Task> GetTodoItem(long id) + { + var todoItem = await _context.TodoItems.FindAsync(id); + + if (todoItem == null) + { + return NotFound(); + } + + return todoItem; + } + + // PUT: api/Todo/5 + // To protect from overposting attacks, please enable the specific properties you want to bind to, for + // more details see https://aka.ms/RazorPagesCRUD. + [HttpPut("{id}")] + public async Task PutTodoItem(long id, TodoItem todoItem) + { + if (id != todoItem.Id) + { + return BadRequest(); + } + + _context.Entry(todoItem).State = EntityState.Modified; + + try + { + await _context.SaveChangesAsync(); + } + catch (DbUpdateConcurrencyException) + { + if (!TodoItemExists(id)) + { + return NotFound(); + } + else + { + throw; + } + } + + return NoContent(); + } + + // POST: api/Todo + // To protect from overposting attacks, please enable the specific properties you want to bind to, for + // more details see https://aka.ms/RazorPagesCRUD. + [HttpPost] + public async Task> PostTodoItem(TodoItem todoItem) + { + _context.TodoItems.Add(todoItem); + await _context.SaveChangesAsync(); + + string instanceId = await _client.StartNewAsync("SetReminder", todoItem.Name); + + return CreatedAtAction("GetTodoItem", new { id = todoItem.Id }, todoItem); + } + + // DELETE: api/Todo/5 + [HttpDelete("{id}")] + public async Task> DeleteTodoItem(long id) + { + var todoItem = await _context.TodoItems.FindAsync(id); + if (todoItem == null) + { + return NotFound(); + } + + _context.TodoItems.Remove(todoItem); + await _context.SaveChangesAsync(); + + return todoItem; + } + + private bool TodoItemExists(long id) + { + return _context.TodoItems.Any(e => e.Id == id); + } + } +} diff --git a/samples/durable-client-managed-identity/aspnetcore-app/Program.cs b/samples/durable-client-managed-identity/aspnetcore-app/Program.cs new file mode 100644 index 000000000..e5f916eef --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/Program.cs @@ -0,0 +1,22 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; + +namespace TodoApi +{ + public class Program + { + + static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/samples/durable-client-managed-identity/aspnetcore-app/README.md b/samples/durable-client-managed-identity/aspnetcore-app/README.md new file mode 100644 index 000000000..4c6cb3567 --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/README.md @@ -0,0 +1,37 @@ +# ASP.NET Core API To Do List Sample with Identity-Based Connection + +This example is adapted from the [To Do List sample](https://github.com/Azure-Samples/dotnet-core-api) in the Azure-Samples repository. It demonstrates an ASP.NET Core application with an injected Durable Client and identity-based connections. In this sample, the Durable Client is configured to use a storage connection with a custom name, `MyStorage`, and is set up to utilize a client secret for authentication. + + +## To make the sample run, you need to: + +1. Create an identity for your Function App in the Azure portal. + +2. Grant the following Role-Based Access Control (RBAC) permissions to the identity: + - Storage Queue Data Contributor + - Storage Blob Data Contributor + - Storage Table Data Contributor + +3. Link your storage account to your Function App by adding either of these two details to your configuration, which is appsettings.json file in this sample . + - accountName + - blobServiceUri, queueServiceUri and tableServiceUri + +4. Add the required identity information to your Functions App configuration, which is appsettings.json file in this sample. + - system-assigned identity: nothing needs to be provided. + - user-assigned identity: + - credential: managedidentity + - clientId + - client secret application: + - clientId + - ClientSecret + - tenantId + + +## Notes +- The storage connection information must be provided in the format specified in the appsettings.json file. +- If your storage information is saved in a custom-named JSON file, be sure to add it to your configuration as shown below. +```csharp +this.Configuration = new ConfigurationBuilder() + .AddJsonFile("myjson.json") + .Build(); +``` \ No newline at end of file diff --git a/samples/durable-client-managed-identity/aspnetcore-app/Startup.cs b/samples/durable-client-managed-identity/aspnetcore-app/Startup.cs new file mode 100644 index 000000000..d0dc745bf --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/Startup.cs @@ -0,0 +1,74 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.EntityFrameworkCore; +using Microsoft.OpenApi.Models; +using TodoApi.Models; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; + +namespace TodoApi +{ + public class Startup + { + public Startup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) + { + // AddDurableClientFactory() registers IDurableClientFactory as a service so the application + // can consume it and and call the Durable Client APIs + services.AddDurableClientFactory(); + + services.AddControllers(); + + // Register the Swagger generator, defining 1 or more Swagger documents + services.AddSwaggerGen(c => + { + c.SwaggerDoc("v1", new OpenApiInfo { Title = "My API", Version = "v1" }); + }); + + services.AddDbContext(options => options.UseInMemoryDatabase("TodoList")); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + // Enable middleware to serve generated Swagger as a JSON endpoint. + app.UseSwagger(); + + // Enable middleware to serve swagger-ui (HTML, JS, CSS, etc.), + // specifying the Swagger JSON endpoint. + app.UseSwaggerUI(c => + { + c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1"); + }); + + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + //app.UseHttpsRedirection(); + + app.UseDefaultFiles(); + + app.UseStaticFiles(); + + app.UseRouting(); + + app.UseAuthorization(); + + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + } + } +} diff --git a/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj new file mode 100644 index 000000000..d37d305c5 --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp3.1 + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.sln b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.sln new file mode 100644 index 000000000..e10bb40b2 --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30503.244 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ToDoList", "ToDoList.csproj", "{D75105D4-B93A-4A9B-B12E-E8EF0F7E6223}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D75105D4-B93A-4A9B-B12E-E8EF0F7E6223}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D75105D4-B93A-4A9B-B12E-E8EF0F7E6223}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D75105D4-B93A-4A9B-B12E-E8EF0F7E6223}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D75105D4-B93A-4A9B-B12E-E8EF0F7E6223}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {E7920E27-E4F7-47B7-B1B9-01F8645883CA} + EndGlobalSection +EndGlobal diff --git a/samples/durable-client-managed-identity/aspnetcore-app/appsettings.json b/samples/durable-client-managed-identity/aspnetcore-app/appsettings.json new file mode 100644 index 000000000..06b8d6289 --- /dev/null +++ b/samples/durable-client-managed-identity/aspnetcore-app/appsettings.json @@ -0,0 +1,17 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*", + "TaskHub": "MyTestHub", + "MyStorage": { + "accountName": "YourStorageAccountName", + "clientId": "", + "clientsecret": "", + "tenantId": "" + } +} \ No newline at end of file diff --git a/samples/durable-client-managed-identity/functions-app/ClientFunction.cs b/samples/durable-client-managed-identity/functions-app/ClientFunction.cs new file mode 100644 index 000000000..cc19a7ca7 --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/ClientFunction.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.ContextImplementations; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; +using Microsoft.Extensions.Configuration; + +namespace DurableClientSampleFunctionApp +{ + public class ClientFunction + { + private readonly IDurableClient _client; + + public ClientFunction(IDurableClientFactory clientFactory, IConfiguration configuration) + { + _client = clientFactory.CreateClient(new DurableClientOptions + { + ConnectionName = "ClientStorage", + TaskHub = configuration["TaskHub"] + }); + } + + [FunctionName("CallHelloSequence")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, + ILogger log) + { + log.LogInformation("C# HTTP trigger function processed a request."); + + string instanceId = await _client.StartNewAsync("E1_HelloSequence"); + + DurableOrchestrationStatus status = await _client.GetStatusAsync(instanceId); + + while (status.RuntimeStatus == OrchestrationRuntimeStatus.Pending || + status.RuntimeStatus == OrchestrationRuntimeStatus.Running || + status.RuntimeStatus == OrchestrationRuntimeStatus.ContinuedAsNew) + { + await Task.Delay(10000); + status = await _client.GetStatusAsync(instanceId); + } + + return new ObjectResult(status); + } + } +} diff --git a/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj b/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj new file mode 100644 index 000000000..34b19725d --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj @@ -0,0 +1,20 @@ + + + netcoreapp3.1 + v3 + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + diff --git a/samples/durable-client-managed-identity/functions-app/README.md b/samples/durable-client-managed-identity/functions-app/README.md new file mode 100644 index 000000000..cfda39c88 --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/README.md @@ -0,0 +1,34 @@ +# Azure Function App with Durable Function and Identity-Based Connection + +This project demonstrates an Azure Function App that invokes a Durable Function through a Durable Client using dependency injection and identity-based connection. In the sample, the function is set up to utilize a storage connection named `Storage` by default. Meanwhile, the integrated Durable Client is set to use a storage connection that is specifically named `ClientStorage`. + + +## To make the sample run, you need to: + +1. Create an identity for your Function App in the Azure portal. + +2. Grant the following Role-Based Access Control (RBAC) permissions to the identity: + - Storage Queue Data Contributor + - Storage Blob Data Contributor + - Storage Table Data Contributor + +3. Link your storage account to your Function App by adding either of these two details to your `local.settings.json` file (for local development) or as environment variables in your Function App settings in Azure. + - \__accountName + - \__blobServiceUri, \__queueServiceUri and \__tableServiceUri + +4. Add the required identity information to your Functions App configuration. + - system-assigned identity: nothing needs to be provided. + - user-assigned identity: + - \__credential: managedidentity + - \__clientId + - client secret application: + - \__clientId + - \__ClientSecret + - \__tenantId + + +## Notes + +- The Azure Functions runtime requires a storage account to start, with the default connection name `Storage`. +- The Durable Client injected also requires a storage account, with the same default connection name `Storage`. However, you can use a custom connection name for a separate storage account as runtime for the durable client. For example, in this sample we use custom name `ClientStorage`. +- To provide the necessary connection information, use the format `__`, as shown in local.settings.json. For example, if you want to specify the accountName, then add the setting `__accountName`. diff --git a/samples/durable-client-managed-identity/functions-app/Startup.cs b/samples/durable-client-managed-identity/functions-app/Startup.cs new file mode 100644 index 000000000..4bb731cfb --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/Startup.cs @@ -0,0 +1,17 @@ +using Microsoft.Azure.Functions.Extensions.DependencyInjection; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; + +[assembly: FunctionsStartup(typeof(DurableClientSampleFunctionApp.Startup))] + +namespace DurableClientSampleFunctionApp +{ + public class Startup : FunctionsStartup + { + public override void Configure(IFunctionsHostBuilder builder) + { + // AddDurableClientFactory() registers IDurableClientFactory as a service so the application + // can consume it and and call the Durable Client APIs + builder.Services.AddDurableClientFactory(); + } + } +} diff --git a/samples/durable-client-managed-identity/functions-app/host.json b/samples/durable-client-managed-identity/functions-app/host.json new file mode 100644 index 000000000..bb3b8dadd --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/host.json @@ -0,0 +1,11 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingExcludedTypes": "Request", + "samplingSettings": { + "isEnabled": true + } + } + } +} \ No newline at end of file diff --git a/samples/durable-client-managed-identity/functions-app/local.settings.json b/samples/durable-client-managed-identity/functions-app/local.settings.json new file mode 100644 index 000000000..a4173f806 --- /dev/null +++ b/samples/durable-client-managed-identity/functions-app/local.settings.json @@ -0,0 +1,9 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage__accountName": "", + "ClientStorage__accountName": "", + "FUNCTIONS_WORKER_RUNTIME": "dotnet", + "TaskHub": "mytesthub" + } +} diff --git a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs index d72cd9b48..b3d180153 100644 --- a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs +++ b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs @@ -621,7 +621,7 @@ public static string IOTypesAnalyzerTitle { } /// - /// Looks up a localized string similar to Method call '{0}' violates the orchestrator deterministic code constraint. Methods definied in source code that are used in an orchestrator must be deterministic.. + /// Looks up a localized string similar to Method call '{0}' violates the orchestrator deterministic code constraint. Methods defined in source code that are used in an orchestrator must be deterministic.. /// public static string MethodAnalyzerMessageFormat { get { @@ -630,7 +630,7 @@ public static string MethodAnalyzerMessageFormat { } /// - /// Looks up a localized string similar to Methods definied in source code that are used in an orchestrator must be deterministic.. + /// Looks up a localized string similar to Methods defined in source code that are used in an orchestrator must be deterministic.. /// public static string MethodAnalyzerTitle { get { diff --git a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx index d40722bfb..1eb6cd2a4 100644 --- a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx +++ b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx @@ -306,10 +306,10 @@ https://docs.microsoft.com/azure/azure-functions/durable/durable-functions-check I/O operations are not allowed inside an orchestrator function. - Method call '{0}' violates the orchestrator deterministic code constraint. Methods definied in source code that are used in an orchestrator must be deterministic. + Method call '{0}' violates the orchestrator deterministic code constraint. Methods defined in source code that are used in an orchestrator must be deterministic. - Methods definied in source code that are used in an orchestrator must be deterministic. + Methods defined in source code that are used in an orchestrator must be deterministic. SignalEntityAsync must use an Entity Interface. diff --git a/src/WebJobs.Extensions.DurableTask.Analyzers/WebJobs.Extensions.DurableTask.Analyzers.csproj b/src/WebJobs.Extensions.DurableTask.Analyzers/WebJobs.Extensions.DurableTask.Analyzers.csproj index be27d32a9..6c627046e 100644 --- a/src/WebJobs.Extensions.DurableTask.Analyzers/WebJobs.Extensions.DurableTask.Analyzers.csproj +++ b/src/WebJobs.Extensions.DurableTask.Analyzers/WebJobs.Extensions.DurableTask.Analyzers.csproj @@ -4,6 +4,7 @@ netstandard2.0 false true + RS1026 diff --git a/src/WebJobs.Extensions.DurableTask/AzureStorageAccountProvider.cs b/src/WebJobs.Extensions.DurableTask/AzureStorageAccountProvider.cs index 1e22e2716..7d42ec413 100644 --- a/src/WebJobs.Extensions.DurableTask/AzureStorageAccountProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/AzureStorageAccountProvider.cs @@ -2,9 +2,11 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using System; +using System.Collections.Concurrent; using DurableTask.AzureStorage; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; using Microsoft.Extensions.Configuration; + #if !FUNCTIONS_V1 using Microsoft.Azure.WebJobs.Extensions.DurableTask.Auth; using Microsoft.WindowsAzure.Storage.Auth; @@ -19,6 +21,9 @@ internal sealed class AzureStorageAccountProvider : IStorageAccountProvider #if !FUNCTIONS_V1 private readonly ITokenCredentialFactory credentialFactory; + private readonly ConcurrentDictionary cachedTokenCredentials = + new ConcurrentDictionary(); + public AzureStorageAccountProvider(IConnectionInfoResolver connectionInfoResolver, ITokenCredentialFactory credentialFactory) { this.connectionInfoResolver = connectionInfoResolver ?? throw new ArgumentNullException(nameof(connectionInfoResolver)); @@ -44,7 +49,9 @@ public StorageAccountDetails GetStorageAccountDetails(string connectionName) AzureStorageAccountOptions account = connectionInfo.Get(); if (account != null) { - TokenCredential credential = this.credentialFactory.Create(connectionInfo); + TokenCredential credential = this.cachedTokenCredentials.GetOrAdd( + connectionName, + attr => this.credentialFactory.Create(connectionInfo)); return new StorageAccountDetails { diff --git a/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs b/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs index 0162d26b4..51015f79e 100644 --- a/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs +++ b/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs @@ -159,7 +159,7 @@ private AzureStorageDurabilityProvider GetAzureStorageStorageProvider(DurableCli // Need to check this.defaultStorageProvider != null for external clients that call GetDurabilityProvider(attribute) // which never initializes the defaultStorageProvider. if (string.Equals(this.defaultSettings?.TaskHubName, settings.TaskHubName, StringComparison.OrdinalIgnoreCase) && - string.Equals(this.defaultSettings?.StorageConnectionString, settings.StorageConnectionString, StringComparison.OrdinalIgnoreCase) && + string.Equals(this.defaultSettings?.StorageAccountDetails?.ConnectionString, settings.StorageAccountDetails?.ConnectionString, StringComparison.OrdinalIgnoreCase) && this.defaultStorageProvider != null) { // It's important that clients use the same AzureStorageOrchestrationService instance @@ -217,6 +217,7 @@ internal AzureStorageOrchestrationServiceSettings GetAzureStorageOrchestrationSe UseSeparateQueueForEntityWorkItems = this.useSeparateQueueForEntityWorkItems, EntityMessageReorderWindowInMinutes = this.options.EntityMessageReorderWindowInMinutes, MaxEntityOperationBatchSize = this.options.MaxEntityOperationBatchSize, + AllowReplayingTerminalInstances = this.azureStorageOptions.AllowReplayingTerminalInstances, }; if (this.inConsumption) diff --git a/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs b/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs index a0c2fddde..499628929 100644 --- a/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs +++ b/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs @@ -43,6 +43,7 @@ public string DurableOrchestrationClientToString(IDurableOrchestrationClient cli ConnectionName = attr.ConnectionName, RpcBaseUrl = localRpcAddress, RequiredQueryStringParameters = this.config.HttpApiHandler.GetUniversalQueryStrings(), + HttpBaseUrl = this.config.HttpApiHandler.GetBaseUrl(), }); } @@ -130,6 +131,14 @@ private class OrchestrationClientInputData /// [JsonProperty("rpcBaseUrl")] public string? RpcBaseUrl { get; set; } + + /// + /// The base URL of the Azure Functions host, used in the out-of-proc model. + /// This URL is sent by the client binding object to the Durable Worker extension, + /// allowing the extension to know the host's base URL for constructing management URLs. + /// + [JsonProperty("httpBaseUrl")] + public string? HttpBaseUrl { get; set; } } } } diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableEntityContext.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableEntityContext.cs index 0b658eca5..228034dd3 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableEntityContext.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableEntityContext.cs @@ -122,13 +122,12 @@ bool IDurableEntityContext.HasState public void CaptureInternalError(Exception e, TaskEntityShim shim) { // first, try to get a quick ETW message out to help us diagnose what happened - string details = Utils.IsFatal(e) ? e.GetType().Name : e.ToString(); this.Config.TraceHelper.EntityBatchFailed( this.HubName, this.Name, this.InstanceId, shim.TraceFlags, - details); + e); // then, record the error for additional reporting and tracking in other places this.InternalError = ExceptionDispatchInfo.Capture(e); @@ -180,22 +179,27 @@ public void ThrowApplicationExceptionsIfAny() } } - public bool ErrorsPresent(out string description) + public bool ErrorsPresent(out string error, out string sanitizedError) { if (this.InternalError != null) { - description = $"Internal error: {this.InternalError.SourceException}"; + error = $"Internal error: {this.InternalError.SourceException}"; + sanitizedError = $"Internal error: {this.InternalError.SourceException.GetType().FullName} \n {this.InternalError.SourceException.StackTrace}"; return true; } else if (this.ApplicationErrors != null) { var messages = this.ApplicationErrors.Select(i => $"({i.SourceException.Message})"); - description = $"One or more operations failed: {string.Concat(messages)}"; + error = $"One or more operations failed: {string.Concat(messages)}"; + + string errorTypes = string.Join(", ", this.ApplicationErrors.Select(i => i.SourceException.GetType().FullName)); + sanitizedError = $"One or more operations failed: {errorTypes}"; return true; } else { - description = string.Empty; + error = string.Empty; + sanitizedError = string.Empty; return false; } } diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs index 572c1b747..782a4c8b1 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs @@ -781,7 +781,7 @@ internal async Task CallDurableTaskFunctionAsync( operationId, operationName, input: "(replayed)", - exception: "(replayed)", + exception: exception, duration: 0, isReplay: true); } @@ -791,7 +791,7 @@ internal async Task CallDurableTaskFunctionAsync( this.Config.Options.HubName, functionName, this.InstanceId, - reason: $"(replayed {exception.GetType().Name})", + exception: exception, functionType: functionType, isReplay: true); } @@ -933,7 +933,7 @@ internal void RaiseEvent(string name, string input) FunctionType.Orchestrator, this.InstanceId, name, - this.Config.GetIntputOutputTrace(responseMessage.Result), + responseMessage.Result, this.IsReplaying); } else @@ -943,7 +943,7 @@ internal void RaiseEvent(string name, string input) this.Name, this.InstanceId, name, - this.Config.GetIntputOutputTrace(input), + input, this.IsReplaying); } diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs index 5a8a50482..86cb21c84 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs @@ -64,18 +64,11 @@ internal OrchestratorExecutionResult GetResult() return this.executionResult ?? throw new InvalidOperationException($"The execution result has not yet been set using {nameof(this.SetResult)}."); } - internal bool TryGetOrchestrationErrorDetails(out string details) + internal bool TryGetOrchestrationErrorDetails(out Exception? failure) { - if (this.failure != null) - { - details = this.failure.Message; - return true; - } - else - { - details = string.Empty; - return false; - } + bool hasError = this.failure != null; + failure = hasError ? this.failure : null; + return hasError; } internal void SetResult(IEnumerable actions, string customStatus) @@ -127,6 +120,31 @@ internal void SetResult(string orchestratorResponseJsonText) this.SetResultInternal(result); } + private void ThrowIfPlatformLevelException(FailureDetails failureDetails) + { + // Recursively inspect the FailureDetails of the failed orchestrator and throw if a platform-level exception is detected. + // + // Today, this method only checks for . In the future, we may want to add more cases. + // Other known platform-level exceptions, like timeouts or process exists due to `Environment.FailFast`, do not yield + // a `OrchestratorExecutionResult` as the isolated invocation is abruptly terminated. Therefore, they don't need to be + // handled in this method. + // However, our tests reveal that OOMs are, surprisngly, caught and returned as a `OrchestratorExecutionResult` + // by the isolated process, and thus need special handling. + // + // It's unclear if all OOMs are caught by the isolated process (probably not), and also if there are other platform-level + // errors that are also caught in the isolated process and returned as a `OrchestratorExecutionResult`. Let's add them + // to this method as we encounter them. + if (failureDetails.InnerFailure?.IsCausedBy() ?? false) + { + throw new SessionAbortedException(failureDetails.ErrorMessage); + } + + if (failureDetails.InnerFailure != null) + { + this.ThrowIfPlatformLevelException(failureDetails.InnerFailure); + } + } + private void SetResultInternal(OrchestratorExecutionResult result) { // Look for an orchestration completion action to see if we need to grab the output. @@ -140,6 +158,14 @@ private void SetResultInternal(OrchestratorExecutionResult result) if (completeAction.OrchestrationStatus == OrchestrationStatus.Failed) { + // If the orchestrator failed due to a platform-level error in the isolated process, + // we should re-throw that exception in the host (this process) invocation pipeline, + // so the invocation can be retried. + if (completeAction.FailureDetails != null) + { + this.ThrowIfPlatformLevelException(completeAction.FailureDetails); + } + string message = completeAction switch { { FailureDetails: { } f } => f.ErrorMessage, diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index 3ae312bef..4e59f460c 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -154,7 +154,7 @@ public DurableTaskExtension( ILogger logger = loggerFactory.CreateLogger(LoggerCategoryName); - this.TraceHelper = new EndToEndTraceHelper(logger, this.Options.Tracing.TraceReplayEvents); + this.TraceHelper = new EndToEndTraceHelper(logger, this.Options.Tracing.TraceReplayEvents, this.Options.Tracing.TraceInputsAndOutputs); this.LifeCycleNotificationHelper = lifeCycleNotificationHelper ?? this.CreateLifeCycleNotificationHelper(); this.durabilityProviderFactory = GetDurabilityProviderFactory(this.Options, logger, orchestrationServiceFactories); this.defaultDurabilityProvider = this.durabilityProviderFactory.GetDurabilityProvider(); @@ -359,7 +359,7 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context) } // Throw if any of the configured options are invalid - this.Options.Validate(this.nameResolver, this.TraceHelper); + this.Options.Validate(this.nameResolver); #pragma warning disable CS0618 // Type or member is obsolete @@ -1037,7 +1037,7 @@ private async Task EntityMiddleware(DispatchMiddlewareContext dispatchContext, F entityContext.HubName, entityContext.Name, entityContext.InstanceId, - this.GetIntputOutputTrace(runtimeState.Input), + runtimeState.Input, FunctionType.Entity, isReplay: false); @@ -1063,13 +1063,14 @@ private async Task EntityMiddleware(DispatchMiddlewareContext dispatchContext, F await next(); // 5. If there were internal or application errors, trace them for DF - if (entityContext.ErrorsPresent(out var description)) + if (entityContext.ErrorsPresent(out string description, out string sanitizedError)) { this.TraceHelper.FunctionFailed( entityContext.HubName, entityContext.Name, entityContext.InstanceId, description, + sanitizedReason: sanitizedError, functionType: FunctionType.Entity, isReplay: false); } @@ -1079,7 +1080,7 @@ private async Task EntityMiddleware(DispatchMiddlewareContext dispatchContext, F entityContext.HubName, entityContext.Name, entityContext.InstanceId, - this.GetIntputOutputTrace(entityContext.State.EntityState), + entityContext.State.EntityState, continuedAsNew: true, functionType: FunctionType.Entity, isReplay: false); @@ -1486,35 +1487,6 @@ bool HasActiveListeners(RegisteredFunctionInfo info) return false; } - internal string GetIntputOutputTrace(string rawInputOutputData) - { - if (this.Options.Tracing.TraceInputsAndOutputs) - { - return rawInputOutputData; - } - else if (rawInputOutputData == null) - { - return "(null)"; - } - else - { - // Azure Storage uses UTF-32 encoding for string payloads - return "(" + Encoding.UTF32.GetByteCount(rawInputOutputData) + " bytes)"; - } - } - - internal string GetExceptionTrace(string rawExceptionData) - { - if (rawExceptionData == null) - { - return "(null)"; - } - else - { - return rawExceptionData; - } - } - /// Task IAsyncConverter.ConvertAsync( HttpRequestMessage request, diff --git a/src/WebJobs.Extensions.DurableTask/EndToEndTraceHelper.cs b/src/WebJobs.Extensions.DurableTask/EndToEndTraceHelper.cs index e730d6e71..8d7e2ddd6 100644 --- a/src/WebJobs.Extensions.DurableTask/EndToEndTraceHelper.cs +++ b/src/WebJobs.Extensions.DurableTask/EndToEndTraceHelper.cs @@ -4,26 +4,31 @@ using System; using System.Diagnostics; using System.Net; +using DurableTask.Core.Common; +using DurableTask.Core.Exceptions; using Microsoft.Extensions.Logging; +#nullable enable namespace Microsoft.Azure.WebJobs.Extensions.DurableTask { internal class EndToEndTraceHelper { private static readonly string ExtensionVersion = FileVersionInfo.GetVersionInfo(typeof(DurableTaskExtension).Assembly.Location).FileVersion; - private static string appName; - private static string slotName; + private static string? appName; + private static string? slotName; private readonly ILogger logger; private readonly bool traceReplayEvents; + private readonly bool shouldTraceRawData; private long sequenceNumber; - public EndToEndTraceHelper(ILogger logger, bool traceReplayEvents) + public EndToEndTraceHelper(ILogger logger, bool traceReplayEvents, bool shouldTraceRawData = false) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.traceReplayEvents = traceReplayEvents; + this.shouldTraceRawData = shouldTraceRawData; } public static string LocalAppName @@ -54,6 +59,54 @@ public static string LocalSlotName #pragma warning disable SA1117 // Parameters should be on same line or separate lines + internal void SanitizeString(string? rawPayload, out string iloggerString, out string durableKustoTableString) + { + string payload = rawPayload ?? string.Empty; + int numCharacters = payload.Length; + string sanitizedPayload = $"(Redacted {numCharacters} characters)"; + + // By default, both ilogger and kusto data should use the sanitized data + iloggerString = sanitizedPayload; + durableKustoTableString = sanitizedPayload; + + // IFF users opts into tracing raw data, then their ILogger gets the raw data + if (this.shouldTraceRawData) + { + iloggerString = payload; + } + } + + internal void SanitizeException(Exception? exception, out string iloggerExceptionString, out string durableKustoTableString) + { + // default case: exception is null + string rawError = string.Empty; + string sanitizedError = string.Empty; + + // if exception is not null + if (exception != null) + { + // common case if exception is not null + rawError = exception.ToString(); + sanitizedError = $"{exception.GetType().FullName}\n{exception.StackTrace}"; + + // if exception is an OrchestrationFailureException, we need to unravel the details + if (exception is OrchestrationFailureException orchestrationFailureException) + { + rawError = orchestrationFailureException.Details; + } + } + + // By default, both ilogger and kusto data should use the sanitized string + iloggerExceptionString = sanitizedError; + durableKustoTableString = sanitizedError; + + // IFF users opts into tracing raw data, then their ILogger gets the raw exception string + if (this.shouldTraceRawData) + { + iloggerExceptionString = rawError; + } + } + public void ExtensionInformationalEvent( string hubName, string instanceId, @@ -126,11 +179,13 @@ public void FunctionStarting( string hubName, string functionName, string instanceId, - string input, + string? input, FunctionType functionType, bool isReplay, int taskEventId = -1) { + this.SanitizeString(input, out string loggerInput, out string sanitizedInput); + if (this.ShouldLogEvent(isReplay)) { EtwEventSource.Instance.FunctionStarting( @@ -140,14 +195,14 @@ public void FunctionStarting( functionName, taskEventId, instanceId, - input, + sanitizedInput, functionType.ToString(), ExtensionVersion, isReplay); this.logger.LogInformation( "{instanceId}: Function '{functionName} ({functionType})' started. IsReplay: {isReplay}. Input: {input}. State: {state}. RuntimeStatus: {runtimeStatus}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}. TaskEventId: {taskEventId}", - instanceId, functionName, functionType, isReplay, input, FunctionState.Started, OrchestrationRuntimeStatus.Running, hubName, + instanceId, functionName, functionType, isReplay, loggerInput, FunctionState.Started, OrchestrationRuntimeStatus.Running, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++, taskEventId); } } @@ -211,12 +266,14 @@ public void FunctionCompleted( string hubName, string functionName, string instanceId, - string output, + string? output, bool continuedAsNew, FunctionType functionType, bool isReplay, int taskEventId = -1) { + this.SanitizeString(output, out string loggerOutput, out string sanitizedOutput); + if (this.ShouldLogEvent(isReplay)) { EtwEventSource.Instance.FunctionCompleted( @@ -226,7 +283,7 @@ public void FunctionCompleted( functionName, taskEventId, instanceId, - output, + sanitizedOutput, continuedAsNew, functionType.ToString(), ExtensionVersion, @@ -234,37 +291,19 @@ public void FunctionCompleted( this.logger.LogInformation( "{instanceId}: Function '{functionName} ({functionType})' completed. ContinuedAsNew: {continuedAsNew}. IsReplay: {isReplay}. Output: {output}. State: {state}. RuntimeStatus: {runtimeStatus}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}. TaskEventId: {taskEventId}", - instanceId, functionName, functionType, continuedAsNew, isReplay, output, FunctionState.Completed, OrchestrationRuntimeStatus.Completed, hubName, + instanceId, functionName, functionType, continuedAsNew, isReplay, loggerOutput, FunctionState.Completed, OrchestrationRuntimeStatus.Completed, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++, taskEventId); } } - public void ProcessingOutOfProcPayload( - string functionName, - string taskHub, - string instanceId, - string details) - { - EtwEventSource.Instance.ProcessingOutOfProcPayload( - functionName, - taskHub, - LocalAppName, - LocalSlotName, - instanceId, - details, - ExtensionVersion); - - this.logger.LogDebug( - "{instanceId}: Function '{functionName} ({functionType})' returned the following OOProc orchestration state: {details}. : {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, FunctionType.Orchestrator, details, taskHub, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); - } - public void FunctionTerminated( string hubName, string functionName, string instanceId, string reason) { + this.SanitizeString(reason, out string loggerReason, out string sanitizedReason); + FunctionType functionType = FunctionType.Orchestrator; EtwEventSource.Instance.FunctionTerminated( @@ -273,14 +312,14 @@ public void FunctionTerminated( LocalSlotName, functionName, instanceId, - reason, + sanitizedReason, functionType.ToString(), ExtensionVersion, IsReplay: false); this.logger.LogWarning( "{instanceId}: Function '{functionName} ({functionType})' was terminated. Reason: {reason}. State: {state}. RuntimeStatus: {runtimeStatus}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, functionType, reason, FunctionState.Terminated, OrchestrationRuntimeStatus.Terminated, hubName, + instanceId, functionName, functionType, loggerReason, FunctionState.Terminated, OrchestrationRuntimeStatus.Terminated, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } @@ -290,6 +329,8 @@ public void SuspendingOrchestration( string instanceId, string reason) { + this.SanitizeString(reason, out string loggerReason, out string sanitizedReason); + FunctionType functionType = FunctionType.Orchestrator; EtwEventSource.Instance.SuspendingOrchestration( @@ -298,14 +339,14 @@ public void SuspendingOrchestration( LocalSlotName, functionName, instanceId, - reason, + sanitizedReason, functionType.ToString(), ExtensionVersion, IsReplay: false); this.logger.LogInformation( "{instanceId}: Suspending function '{functionName} ({functionType})'. Reason: {reason}. State: {state}. RuntimeStatus: {runtimeStatus}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, functionType, reason, FunctionState.Suspended, OrchestrationRuntimeStatus.Suspended, hubName, + instanceId, functionName, functionType, loggerReason, FunctionState.Suspended, OrchestrationRuntimeStatus.Suspended, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } @@ -315,6 +356,8 @@ public void ResumingOrchestration( string instanceId, string reason) { + this.SanitizeString(reason, out string loggerReason, out string sanitizedReason); + FunctionType functionType = FunctionType.Orchestrator; EtwEventSource.Instance.ResumingOrchestration( @@ -323,14 +366,14 @@ public void ResumingOrchestration( LocalSlotName, functionName, instanceId, - reason, + sanitizedReason, functionType.ToString(), ExtensionVersion, IsReplay: false); this.logger.LogInformation( "{instanceId}: Resuming function '{functionName} ({functionType})'. Reason: {reason}. State: {state}. RuntimeStatus: {runtimeStatus}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, functionType, reason, FunctionState.Scheduled, OrchestrationRuntimeStatus.Running, hubName, + instanceId, functionName, functionType, loggerReason, FunctionState.Scheduled, OrchestrationRuntimeStatus.Running, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } @@ -340,6 +383,8 @@ public void FunctionRewound( string instanceId, string reason) { + this.SanitizeString(reason, out string loggerReason, out string sanitizedReason); + FunctionType functionType = FunctionType.Orchestrator; EtwEventSource.Instance.FunctionRewound( @@ -348,22 +393,36 @@ public void FunctionRewound( LocalSlotName, functionName, instanceId, - reason, + sanitizedReason, functionType.ToString(), ExtensionVersion, IsReplay: false); this.logger.LogWarning( "{instanceId}: Function '{functionName} ({functionType})' was rewound. Reason: {reason}. State: {state}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, functionType, reason, FunctionState.Rewound, hubName, + instanceId, functionName, functionType, loggerReason, FunctionState.Rewound, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } + public void FunctionFailed( + string hubName, + string functionName, + string instanceId, + Exception? exception, + FunctionType functionType, + bool isReplay, + int taskEventId = -1) + { + this.SanitizeException(exception, out string loggerReason, out string sanitizedReason); + this.FunctionFailed(hubName, functionName, instanceId, loggerReason, sanitizedReason, functionType, isReplay, taskEventId); + } + public void FunctionFailed( string hubName, string functionName, string instanceId, string reason, + string sanitizedReason, FunctionType functionType, bool isReplay, int taskEventId = -1) @@ -377,7 +436,7 @@ public void FunctionFailed( functionName, taskEventId, instanceId, - reason, + sanitizedReason, functionType.ToString(), ExtensionVersion, isReplay); @@ -424,6 +483,9 @@ public void OperationCompleted( double duration, bool isReplay) { + this.SanitizeString(input, out string loggerInput, out string sanitizedInput); + this.SanitizeString(output, out string loggerOutput, out string sanitizedOutput); + if (this.ShouldLogEvent(isReplay)) { EtwEventSource.Instance.OperationCompleted( @@ -434,8 +496,8 @@ public void OperationCompleted( instanceId, operationId, operationName, - input, - output, + sanitizedInput, + sanitizedOutput, duration, FunctionType.Entity.ToString(), ExtensionVersion, @@ -443,11 +505,27 @@ public void OperationCompleted( this.logger.LogInformation( "{instanceId}: Function '{functionName} ({functionType})' completed '{operationName}' operation {operationId} in {duration}ms. IsReplay: {isReplay}. Input: {input}. Output: {output}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, FunctionType.Entity, operationName, operationId, duration, isReplay, input, output, + instanceId, functionName, FunctionType.Entity, operationName, operationId, duration, isReplay, loggerInput, loggerOutput, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } } + public void OperationFailed( + string hubName, + string functionName, + string instanceId, + string operationId, + string operationName, + string input, + Exception exception, + double duration, + bool isReplay) + { + this.SanitizeString(input, out string loggerInput, out string sanitizedInput); + this.SanitizeException(exception, out string loggerException, out string sanitizedException); + this.OperationFailed(hubName, functionName, instanceId, operationId, operationName, sanitizedInput, loggerInput, sanitizedException, loggerException, duration, isReplay); + } + public void OperationFailed( string hubName, string functionName, @@ -458,6 +536,24 @@ public void OperationFailed( string exception, double duration, bool isReplay) + { + this.SanitizeString(input, out string loggerInput, out string sanitizedInput); + this.SanitizeString(exception, out string loggerException, out string sanitizedException); + this.OperationFailed(hubName, functionName, instanceId, operationId, operationName, sanitizedInput, loggerInput, sanitizedException, loggerException, duration, isReplay); + } + + private void OperationFailed( + string hubName, + string functionName, + string instanceId, + string operationId, + string operationName, + string sanitizedInput, + string loggerInput, + string sanitizedException, + string loggerException, + double duration, + bool isReplay) { if (this.ShouldLogEvent(isReplay)) { @@ -469,8 +565,8 @@ public void OperationFailed( instanceId, operationId, operationName, - input, - exception, + sanitizedInput, + sanitizedException, duration, FunctionType.Entity.ToString(), ExtensionVersion, @@ -478,7 +574,7 @@ public void OperationFailed( this.logger.LogError( "{instanceId}: Function '{functionName} ({functionType})' failed '{operationName}' operation {operationId} after {duration}ms with exception {exception}. Input: {input}. IsReplay: {isReplay}. HubName: {hubName}. AppName: {appName}. SlotName: {slotName}. ExtensionVersion: {extensionVersion}. SequenceNumber: {sequenceNumber}.", - instanceId, functionName, FunctionType.Entity, operationName, operationId, duration, exception, input, isReplay, hubName, + instanceId, functionName, FunctionType.Entity, operationName, operationId, duration, loggerException, loggerInput, isReplay, hubName, LocalAppName, LocalSlotName, ExtensionVersion, this.sequenceNumber++); } } @@ -491,6 +587,8 @@ public void ExternalEventRaised( string input, bool isReplay) { + this.SanitizeString(input, out string _, out string sanitizedInput); + if (this.ShouldLogEvent(isReplay)) { FunctionType functionType = FunctionType.Orchestrator; @@ -502,7 +600,7 @@ public void ExternalEventRaised( functionName, instanceId, eventName, - input, + sanitizedInput, functionType.ToString(), ExtensionVersion, isReplay); @@ -608,6 +706,8 @@ public void EntityResponseReceived( string result, bool isReplay) { + this.SanitizeString(result, out string _, out string sanitizedResult); + if (this.ShouldLogEvent(isReplay)) { EtwEventSource.Instance.EntityResponseReceived( @@ -617,7 +717,7 @@ public void EntityResponseReceived( functionName, instanceId, operationId, - result, + sanitizedResult, functionType.ToString(), ExtensionVersion, isReplay); @@ -806,9 +906,11 @@ public void EntityBatchFailed( string functionName, string instanceId, string traceFlags, - string details) + Exception error) { FunctionType functionType = FunctionType.Entity; + string details = Utils.IsFatal(error) ? error.GetType().Name : error.ToString(); + string sanitizedDetails = $"{error.GetType().FullName}\n{error.StackTrace}"; EtwEventSource.Instance.EntityBatchFailed( hubName, @@ -817,7 +919,7 @@ public void EntityBatchFailed( functionName, instanceId, traceFlags, - details, + sanitizedDetails, functionType.ToString(), ExtensionVersion); diff --git a/src/WebJobs.Extensions.DurableTask/EtwEventSource.cs b/src/WebJobs.Extensions.DurableTask/EtwEventSource.cs index 77f9bb0e4..019e27a2f 100644 --- a/src/WebJobs.Extensions.DurableTask/EtwEventSource.cs +++ b/src/WebJobs.Extensions.DurableTask/EtwEventSource.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See LICENSE in the project root for license information. - -using System; +#nullable enable using System.Diagnostics.Tracing; namespace Microsoft.Azure.WebJobs.Extensions.DurableTask @@ -43,7 +42,7 @@ public void FunctionStarting( string FunctionName, int TaskEventId, string InstanceId, - string Input, + string? Input, string FunctionType, string ExtensionVersion, bool IsReplay) @@ -88,7 +87,7 @@ public void ExternalEventRaised( string FunctionName, string InstanceId, string EventName, - string Input, + string? Input, string FunctionType, string ExtensionVersion, bool IsReplay) @@ -104,7 +103,7 @@ public void FunctionCompleted( string FunctionName, int TaskEventId, string InstanceId, - string Output, + string? Output, bool ContinuedAsNew, string FunctionType, string ExtensionVersion, @@ -204,7 +203,7 @@ public void EventGridNotificationException( string SlotName, string FunctionName, FunctionState FunctionState, - string Version, + string? Version, string InstanceId, string Details, string Reason, @@ -222,8 +221,8 @@ public void ExtensionInformationalEvent( string TaskHub, string AppName, string SlotName, - string FunctionName, - string InstanceId, + string? FunctionName, + string? InstanceId, string Details, string ExtensionVersion) { @@ -235,8 +234,8 @@ public void ExtensionWarningEvent( string TaskHub, string AppName, string SlotName, - string FunctionName, - string InstanceId, + string? FunctionName, + string? InstanceId, string Details, string ExtensionVersion) { @@ -265,7 +264,7 @@ public void FunctionRewound( string SlotName, string FunctionName, string InstanceId, - string Reason, + string? Reason, string FunctionType, string ExtensionVersion, bool IsReplay) @@ -297,7 +296,7 @@ public void EntityResponseReceived( string FunctionName, string InstanceId, string OperationId, - string Result, + string? Result, string FunctionType, string ExtensionVersion, bool IsReplay) @@ -313,7 +312,7 @@ public void EntityLockAcquired( string FunctionName, string InstanceId, string RequestingInstanceId, - string RequestingExecutionId, + string? RequestingExecutionId, string RequestId, string FunctionType, string ExtensionVersion, @@ -347,8 +346,8 @@ public void OperationCompleted( string InstanceId, string OperationId, string OperationName, - string Input, - string Output, + string? Input, + string? Output, double Duration, string FunctionType, string ExtensionVersion, @@ -366,7 +365,7 @@ public void OperationFailed( string InstanceId, string OperationId, string OperationName, - string Input, + string? Input, string Exception, double Duration, string FunctionType, diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 538f473ee..81253c399 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -1241,7 +1241,9 @@ private HttpResponseMessage CreateCheckStatusResponseMessage( internal Uri GetWebhookUri() { - return this.webhookUrlProvider?.Invoke() ?? throw new InvalidOperationException("Webhooks are not configured"); + string errorMessage = "Webhooks are not configured. This may occur if the environment variable `WEBSITE_HOSTNAME` is not set (should be automatically set for Azure Functions). " + + "Try setting it to the appropiate URI to reach your app. For example: the DNS name of the app, or a value of the form :."; + return this.webhookUrlProvider?.Invoke() ?? throw new InvalidOperationException(errorMessage); } internal bool TryGetRpcBaseUrl(out Uri rpcBaseUrl) diff --git a/src/WebJobs.Extensions.DurableTask/Listener/TaskActivityShim.cs b/src/WebJobs.Extensions.DurableTask/Listener/TaskActivityShim.cs index 9481a0be0..128ed4c44 100644 --- a/src/WebJobs.Extensions.DurableTask/Listener/TaskActivityShim.cs +++ b/src/WebJobs.Extensions.DurableTask/Listener/TaskActivityShim.cs @@ -10,6 +10,7 @@ using Microsoft.Azure.WebJobs.Host; using Microsoft.Azure.WebJobs.Host.Executors; +#nullable enable namespace Microsoft.Azure.WebJobs.Extensions.DurableTask { /// @@ -58,7 +59,7 @@ public override async Task RunAsync(TaskContext context, string rawInput this.config.Options.HubName, this.activityName, instanceId, - this.config.GetIntputOutputTrace(rawInput), + rawInput, functionType: FunctionType.Activity, isReplay: false, taskEventId: this.taskEventId); @@ -76,7 +77,7 @@ public override async Task RunAsync(TaskContext context, string rawInput this.config.Options.HubName, this.activityName, instanceId, - this.config.GetIntputOutputTrace(serializedOutput), + serializedOutput, continuedAsNew: false, functionType: FunctionType.Activity, isReplay: false, @@ -100,7 +101,7 @@ public override async Task RunAsync(TaskContext context, string rawInput case WrappedFunctionResult.FunctionResultStatus.FunctionTimeoutError: // Flow the original activity function exception to the orchestration // without the outer FunctionInvocationException. - Exception exceptionToReport = StripFunctionInvocationException(result.Exception); + Exception? exceptionToReport = StripFunctionInvocationException(result.Exception); if (OutOfProcExceptionHelpers.TryGetExceptionWithFriendlyMessage( exceptionToReport, @@ -113,13 +114,13 @@ public override async Task RunAsync(TaskContext context, string rawInput this.config.Options.HubName, this.activityName, instanceId, - exceptionToReport?.ToString() ?? string.Empty, + exceptionToReport, functionType: FunctionType.Activity, isReplay: false, taskEventId: this.taskEventId); throw new TaskFailureException( - $"Activity function '{this.activityName}' failed: {exceptionToReport.Message}", + $"Activity function '{this.activityName}' failed: {exceptionToReport!.Message}", Utils.SerializeCause(exceptionToReport, this.config.ErrorDataConverter)); default: // we throw a TaskFailureException to ensure deserialization is possible. @@ -143,7 +144,7 @@ internal void SetTaskEventId(int taskEventId) this.taskEventId = taskEventId; } - private static Exception StripFunctionInvocationException(Exception e) + private static Exception? StripFunctionInvocationException(Exception? e) { var infrastructureException = e as FunctionInvocationException; if (infrastructureException?.InnerException != null) diff --git a/src/WebJobs.Extensions.DurableTask/Listener/TaskEntityShim.cs b/src/WebJobs.Extensions.DurableTask/Listener/TaskEntityShim.cs index 1e0998ce4..648a86731 100644 --- a/src/WebJobs.Extensions.DurableTask/Listener/TaskEntityShim.cs +++ b/src/WebJobs.Extensions.DurableTask/Listener/TaskEntityShim.cs @@ -237,7 +237,7 @@ public override async Task Execute(OrchestrationContext innerContext, st this.context.Name, this.context.InstanceId, this.entityTraceInfo.TraceFlags, - this.context.InternalError.ToString()); + this.context.InternalError.SourceException); } else { @@ -533,8 +533,8 @@ private async Task ProcessOperationRequestAsync(RequestMessage request) this.context.InstanceId, request.Id.ToString(), request.Operation, - this.Config.GetIntputOutputTrace(this.context.RawInput), - this.Config.GetIntputOutputTrace(response.Result), + this.context.RawInput, + response.Result, stopwatch.Elapsed.TotalMilliseconds, isReplay: false); } @@ -546,8 +546,8 @@ private async Task ProcessOperationRequestAsync(RequestMessage request) this.context.InstanceId, request.Id.ToString(), request.Operation, - this.Config.GetIntputOutputTrace(this.context.RawInput), - exception.ToString(), + this.context.RawInput, + exception, stopwatch.Elapsed.TotalMilliseconds, isReplay: false); } @@ -638,8 +638,8 @@ private async Task ExecuteOutOfProcBatch() this.context.InstanceId, request.Id.ToString(), request.Operation, - this.Config.GetIntputOutputTrace(request.Input), - this.Config.GetIntputOutputTrace(result.Result), + request.Input, + result.Result, result.DurationInMilliseconds, isReplay: false); } @@ -654,8 +654,8 @@ private async Task ExecuteOutOfProcBatch() this.context.InstanceId, request.Id.ToString(), request.Operation, - this.Config.GetIntputOutputTrace(request.Input), - this.Config.GetIntputOutputTrace(result.Result), + request.Input, + result.Result, result.DurationInMilliseconds, isReplay: false); } diff --git a/src/WebJobs.Extensions.DurableTask/Listener/TaskOrchestrationShim.cs b/src/WebJobs.Extensions.DurableTask/Listener/TaskOrchestrationShim.cs index 8cb6a33ff..3f127dca5 100644 --- a/src/WebJobs.Extensions.DurableTask/Listener/TaskOrchestrationShim.cs +++ b/src/WebJobs.Extensions.DurableTask/Listener/TaskOrchestrationShim.cs @@ -75,7 +75,7 @@ public override async Task Execute(OrchestrationContext innerContext, st this.context.HubName, this.context.Name, this.context.InstanceId, - this.Config.GetIntputOutputTrace(serializedInput), + serializedInput, FunctionType.Orchestrator, this.context.IsReplaying); status = OrchestrationRuntimeStatus.Running; @@ -113,7 +113,7 @@ public override async Task Execute(OrchestrationContext innerContext, st this.context.HubName, this.context.Name, this.context.InstanceId, - this.Config.GetIntputOutputTrace(serializedOutput), + serializedOutput, this.context.ContinuedAsNew, FunctionType.Orchestrator, this.context.IsReplaying); @@ -184,14 +184,14 @@ private async Task InvokeUserCodeAndHandleResults( } catch (OrchestrationFailureException ex) { - this.TraceAndSendExceptionNotification(ex.Details); + this.TraceAndSendExceptionNotification(ex); this.context.OrchestrationException = ExceptionDispatchInfo.Capture(ex); throw ex; } } else { - this.TraceAndSendExceptionNotification(e.ToString()); + this.TraceAndSendExceptionNotification(e); var orchestrationException = new OrchestrationFailureException( $"Orchestrator function '{this.context.Name}' failed: {e.Message}", Utils.SerializeCause(e, innerContext.ErrorDataConverter)); @@ -212,13 +212,19 @@ private async Task InvokeUserCodeAndHandleResults( } } - private void TraceAndSendExceptionNotification(string exceptionDetails) + private void TraceAndSendExceptionNotification(Exception exception) { + string exceptionDetails = exception.Message; + if (exception is OrchestrationFailureException orchestrationFailureException) + { + exceptionDetails = orchestrationFailureException.Details; + } + this.config.TraceHelper.FunctionFailed( this.context.HubName, this.context.Name, this.context.InstanceId, - exceptionDetails, + exception: exception, FunctionType.Orchestrator, this.context.IsReplaying); diff --git a/src/WebJobs.Extensions.DurableTask/Listener/WrappedFunctionResult.cs b/src/WebJobs.Extensions.DurableTask/Listener/WrappedFunctionResult.cs index fade92f13..f758179d2 100644 --- a/src/WebJobs.Extensions.DurableTask/Listener/WrappedFunctionResult.cs +++ b/src/WebJobs.Extensions.DurableTask/Listener/WrappedFunctionResult.cs @@ -3,13 +3,14 @@ using System; +#nullable enable namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Listener { internal class WrappedFunctionResult { private WrappedFunctionResult( FunctionResultStatus status, - Exception ex) + Exception? ex) { this.Exception = ex; this.ExecutionStatus = status; @@ -24,7 +25,7 @@ internal enum FunctionResultStatus FunctionsHostStoppingError = 4, // host was shutting down; treated as a functions runtime error } - internal Exception Exception { get; } + internal Exception? Exception { get; } internal FunctionResultStatus ExecutionStatus { get; } diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml index 0088042e9..cdf5d8700 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml @@ -93,6 +93,13 @@ HTTP endpoint. For out-of-proc "v2" (middelware passthrough), this is a gRPC endpoint. + + + The base URL of the Azure Functions host, used in the out-of-proc model. + This URL is sent by the client binding object to the Durable Worker extension, + allowing the extension to know the host's base URL for constructing management URLs. + + The result of a clean entity storage operation. @@ -4208,6 +4215,20 @@ A boolean indicating whether to use the table partition strategy. Defaults to false. + + + When false, when an orchestrator is in a terminal state (e.g. Completed, Failed, Terminated), events for that orchestrator are discarded. + Otherwise, events for a terminal orchestrator induce a replay. This may be used to recompute the state of the orchestrator in the "Instances Table". + + + Transactions across Azure Tables are not possible, so we independently update the "History table" and then the "Instances table" + to set the state of the orchestrator. + If a crash were to occur between these two updates, the state of the orchestrator in the "Instances table" would be incorrect. + By setting this configuration to true, you can recover from these inconsistencies by forcing a replay of the orchestrator in response + to a client event like a termination request or an external event, which gives the framework another opportunity to update the state of + the orchestrator in the "Instances table". To force a replay after enabling this configuration, just send any external event to the affected instanceId. + + Throws an exception if the provided hub name violates any naming conventions for the storage provider. diff --git a/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs index 1667aabaf..4a6a506cb 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs @@ -179,6 +179,20 @@ public string TrackingStoreConnectionStringName /// A boolean indicating whether to use the table partition strategy. Defaults to false. public bool UseTablePartitionManagement { get; set; } = false; + /// + /// When false, when an orchestrator is in a terminal state (e.g. Completed, Failed, Terminated), events for that orchestrator are discarded. + /// Otherwise, events for a terminal orchestrator induce a replay. This may be used to recompute the state of the orchestrator in the "Instances Table". + /// + /// + /// Transactions across Azure Tables are not possible, so we independently update the "History table" and then the "Instances table" + /// to set the state of the orchestrator. + /// If a crash were to occur between these two updates, the state of the orchestrator in the "Instances table" would be incorrect. + /// By setting this configuration to true, you can recover from these inconsistencies by forcing a replay of the orchestrator in response + /// to a client event like a termination request or an external event, which gives the framework another opportunity to update the state of + /// the orchestrator in the "Instances table". To force a replay after enabling this configuration, just send any external event to the affected instanceId. + /// + public bool AllowReplayingTerminalInstances { get; set; } = false; + /// /// Throws an exception if the provided hub name violates any naming conventions for the storage provider. /// diff --git a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs index bc555386f..c7322044e 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs @@ -302,7 +302,7 @@ internal void TraceConfiguration(EndToEndTraceHelper traceHelper, JObject storag traceHelper.TraceConfiguration(this.HubName, configurationJson.ToString(Formatting.None)); } - internal void Validate(INameResolver environmentVariableResolver, EndToEndTraceHelper traceHelper) + internal void Validate(INameResolver environmentVariableResolver) { if (string.IsNullOrEmpty(this.HubName)) { @@ -320,12 +320,9 @@ internal void Validate(INameResolver environmentVariableResolver, EndToEndTraceH runtimeLanguage != null && // If we don't know from the environment variable, don't assume customer isn't .NET !string.Equals(runtimeLanguage, "dotnet", StringComparison.OrdinalIgnoreCase)) { - traceHelper.ExtensionWarningEvent( - hubName: this.HubName, - functionName: string.Empty, - instanceId: string.Empty, - message: "Durable Functions does not work with extendedSessions = true for non-.NET languages. This value is being set to false instead. See https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-perf-and-scale#extended-sessions for more details."); - this.ExtendedSessionsEnabled = false; + throw new InvalidOperationException( + "Durable Functions with extendedSessionsEnabled set to 'true' is only supported when using the in-process .NET worker. Please remove the setting or change it to 'false'." + + "See https://docs.microsoft.com/azure/azure-functions/durable/durable-functions-perf-and-scale#extended-sessions for more details."); } this.Notifications.Validate(); diff --git a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs index 8a7983cba..88a7612dc 100644 --- a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs +++ b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs @@ -95,7 +95,7 @@ public async Task CallOrchestratorAsync(DispatchMiddlewareContext dispatchContex this.Options.HubName, functionName.Name, instance.InstanceId, - isReplaying ? "(replay)" : this.extension.GetIntputOutputTrace(startEvent.Input), + startEvent.Input, FunctionType.Orchestrator, isReplaying); @@ -138,10 +138,15 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync( byte[] triggerReturnValueBytes = Convert.FromBase64String(triggerReturnValue); P.OrchestratorResponse response = P.OrchestratorResponse.Parser.ParseFrom(triggerReturnValueBytes); + + // TrySetResult may throw if a platform-level error is encountered (like an out of memory exception). context.SetResult( response.Actions.Select(ProtobufUtils.ToOrchestratorAction), response.CustomStatus); + // Here we throw if the orchestrator completed with an application-level error. When we do this, + // the function's result type will be of type `OrchestrationFailureException` which is reserved + // for application-level errors that do not need to be re-tried. context.ThrowIfFailed(); }, #pragma warning restore CS0618 // Type or member is obsolete (not intended for general public use) @@ -159,6 +164,19 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync( // Re-throw so we can abort this invocation. this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested(); } + + // we abort the invocation on "platform level errors" such as: + // - a timeout + // - an out of memory exception + // - a worker process exit + if (functionResult.Exception is Host.FunctionTimeoutException + || functionResult.Exception?.InnerException is SessionAbortedException // see RemoteOrchestrationContext.TrySetResultInternal for details on OOM-handling + || (functionResult.Exception?.InnerException?.GetType().ToString().Contains("WorkerProcessExitException") ?? false)) + { + // TODO: the `WorkerProcessExitException` type is not exposed in our dependencies, it's part of WebJobs.Host.Script. + // Should we add that dependency or should it be exposed in WebJobs.Host? + throw functionResult.Exception; + } } catch (Exception hostRuntimeException) { @@ -188,7 +206,7 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync( this.Options.HubName, functionName.Name, instance.InstanceId, - this.extension.GetIntputOutputTrace(context.SerializedOutput), + context.SerializedOutput, context.ContinuedAsNew, FunctionType.Orchestrator, isReplay: false); @@ -214,7 +232,7 @@ await this.LifeCycleNotificationHelper.OrchestratorCompletedAsync( isReplay: false); } } - else if (context.TryGetOrchestrationErrorDetails(out string details)) + else if (context.TryGetOrchestrationErrorDetails(out Exception? exception)) { // the function failed because the orchestrator failed. @@ -224,7 +242,7 @@ await this.LifeCycleNotificationHelper.OrchestratorCompletedAsync( this.Options.HubName, functionName.Name, instance.InstanceId, - details, + exception, FunctionType.Orchestrator, isReplay: false); @@ -232,20 +250,19 @@ await this.LifeCycleNotificationHelper.OrchestratorFailedAsync( this.Options.HubName, functionName.Name, instance.InstanceId, - details, + exception?.Message ?? string.Empty, isReplay: false); } else { // the function failed for some other reason - - string exceptionDetails = functionResult.Exception.ToString(); + string exceptionDetails = functionResult.Exception?.ToString() ?? "Framework-internal message: exception details could not be extracted"; this.TraceHelper.FunctionFailed( this.Options.HubName, functionName.Name, instance.InstanceId, - exceptionDetails, + functionResult.Exception, FunctionType.Orchestrator, isReplay: false); @@ -258,7 +275,7 @@ await this.LifeCycleNotificationHelper.OrchestratorFailedAsync( orchestratorResult = OrchestratorExecutionResult.ForFailure( message: $"Function '{functionName}' failed with an unhandled exception.", - functionResult.Exception); + functionResult.Exception ?? new Exception($"Function '{functionName}' failed with an unknown unhandled exception")); } // Send the result of the orchestrator function to the DTFx dispatch pipeline. @@ -320,7 +337,7 @@ void SetErrorResult(FailureDetails failureDetails) this.Options.HubName, functionName.Name, batchRequest.InstanceId, - this.extension.GetIntputOutputTrace(batchRequest.EntityState), + batchRequest.EntityState, functionType: FunctionType.Entity, isReplay: false); @@ -396,7 +413,7 @@ void SetErrorResult(FailureDetails failureDetails) this.Options.HubName, functionName.Name, batchRequest.InstanceId, - functionResult.Exception.ToString(), + functionResult.Exception, FunctionType.Entity, isReplay: false); @@ -429,7 +446,7 @@ void SetErrorResult(FailureDetails failureDetails) this.Options.HubName, functionName.Name, batchRequest.InstanceId, - this.extension.GetIntputOutputTrace(batchRequest.EntityState), + batchRequest.EntityState, batchResult.EntityState != null, FunctionType.Entity, isReplay: false); @@ -496,7 +513,7 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F this.Options.HubName, functionName.Name, instance.InstanceId, - this.extension.GetIntputOutputTrace(rawInput), + rawInput, functionType: FunctionType.Activity, isReplay: false, taskEventId: scheduledEvent.EventId); @@ -542,7 +559,7 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F this.Options.HubName, functionName.Name, instance.InstanceId, - this.extension.GetIntputOutputTrace(serializedOutput), + serializedOutput, continuedAsNew: false, FunctionType.Activity, isReplay: false, @@ -562,7 +579,7 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F this.Options.HubName, functionName.Name, instance.InstanceId, - result.Exception.ToString(), + result.Exception, FunctionType.Activity, isReplay: false, scheduledEvent.EventId); diff --git a/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs b/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs index 57b84012b..d55bce846 100644 --- a/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs +++ b/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs @@ -97,7 +97,11 @@ public static P.HistoryEvent ToHistoryEventProto(HistoryEvent e) }, }, ScheduledStartTimestamp = startedEvent.ScheduledStartTime == null ? null : Timestamp.FromDateTime(startedEvent.ScheduledStartTime.Value), - CorrelationData = startedEvent.Correlation, + ParentTraceContext = startedEvent.ParentTraceContext == null ? null : new P.TraceContext + { + TraceParent = startedEvent.ParentTraceContext.TraceParent, + TraceState = startedEvent.ParentTraceContext.TraceState, + }, }; break; case EventType.ExecutionTerminated: diff --git a/src/WebJobs.Extensions.DurableTask/StandardConnectionInfoProvider.cs b/src/WebJobs.Extensions.DurableTask/StandardConnectionInfoProvider.cs index 0a9bcca7d..2597877bf 100644 --- a/src/WebJobs.Extensions.DurableTask/StandardConnectionInfoProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/StandardConnectionInfoProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using System; +using System.Linq; using Microsoft.Extensions.Configuration; namespace Microsoft.Azure.WebJobs.Extensions.DurableTask @@ -23,10 +24,40 @@ public StandardConnectionInfoProvider(IConfiguration configuration) this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); } + // This implementation is a clone of `IConfigurationSection.Exists` found here https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Configuration.Abstractions/src/ConfigurationExtensions.cs#L78 + // Functions host v1 (.net462 framework) doesn't support this method so we implement a substitute one here. + private bool IfExists(IConfigurationSection section) + { + if (section == null) + { + return false; + } + + if (section.Value == null) + { + return section.GetChildren().Any(); + } + + return true; + } + /// public IConfigurationSection Resolve(string name) { - return this.configuration.GetSection(name); + // This implementation is a replica of the WebJobsConnectionInfoProvider used for the internal durable client. + // The original code can be found at: + // https://github.com/Azure/azure-functions-durable-extension/blob/dev/src/WebJobs.Extensions.DurableTask/WebJobsConnectionInfoProvider.cs#L37. + // We need to first check the configuration section with the AzureWebJobs prefix, as this is the default name within the Functions app whether it's internal or external. + string prefixedConnectionStringName = "AzureWebJobs" + name; + IConfigurationSection section = this.configuration?.GetSection(prefixedConnectionStringName); + + if (!this.IfExists(section)) + { + // If the section doesn't exist, then look for the configuration section without the prefix, since there is no prefix outside the WebJobs app. + section = this.configuration?.GetSection(name); + } + + return section; } } -} \ No newline at end of file +} diff --git a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj index 5876cc724..05b1a7899 100644 --- a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj +++ b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj @@ -6,7 +6,7 @@ Microsoft.Azure.WebJobs.Extensions.DurableTask 2 13 - 3 + 7 $(PackageSuffix) $(MajorVersion).$(MinorVersion).$(PatchVersion) $(MajorVersion).0.0.0 @@ -57,7 +57,7 @@ - + @@ -75,7 +75,7 @@ $(AssemblyName).xml - + @@ -96,7 +96,7 @@ $(DefineConstants);FUNCTIONS_V2_OR_GREATER;FUNCTIONS_V3_OR_GREATER - + @@ -107,14 +107,14 @@ - + - - + + diff --git a/src/Worker.Extensions.DurableTask/AssemblyInfo.cs b/src/Worker.Extensions.DurableTask/AssemblyInfo.cs index 13dabba38..0be4e12df 100644 --- a/src/Worker.Extensions.DurableTask/AssemblyInfo.cs +++ b/src/Worker.Extensions.DurableTask/AssemblyInfo.cs @@ -5,5 +5,5 @@ using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; // TODO: Find a way to generate this dynamically at build-time -[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.DurableTask", "2.13.3")] -[assembly: InternalsVisibleTo("Worker.Extensions.DurableTask.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] \ No newline at end of file +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.DurableTask", "2.13.7")] +[assembly: InternalsVisibleTo("Worker.Extensions.DurableTask.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] diff --git a/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs b/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs index 1d3da9003..2cfc2706e 100644 --- a/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs +++ b/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs @@ -49,7 +49,7 @@ public ValueTask ConvertAsync(ConverterContext context) } DurableTaskClient client = this.clientProvider.GetClient(endpoint, inputData?.taskHubName, inputData?.connectionName); - client = new FunctionsDurableTaskClient(client, inputData!.requiredQueryStringParameters); + client = new FunctionsDurableTaskClient(client, inputData!.requiredQueryStringParameters, inputData!.httpBaseUrl); return new ValueTask(ConversionResult.Success(client)); } catch (Exception innerException) @@ -62,5 +62,5 @@ public ValueTask ConvertAsync(ConverterContext context) } // Serializer is case-sensitive and incoming JSON properties are camel-cased. - private record DurableClientInputData(string rpcBaseUrl, string taskHubName, string connectionName, string requiredQueryStringParameters); + private record DurableClientInputData(string rpcBaseUrl, string taskHubName, string connectionName, string requiredQueryStringParameters, string httpBaseUrl); } diff --git a/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs b/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs index 286c206fe..251ebb2d7 100644 --- a/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs +++ b/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -18,6 +19,70 @@ namespace Microsoft.Azure.Functions.Worker; /// public static class DurableTaskClientExtensions { + /// + /// Waits for the completion of the specified orchestration instance with a retry interval, controlled by the cancellation token. + /// If the orchestration does not complete within the required time, returns an HTTP response containing the class to manage instances. + /// + /// The . + /// The HTTP request that this response is for. + /// The ID of the orchestration instance to check. + /// The timeout between checks for output from the durable function. The default value is 1 second. + /// Optional parameter that configures the http response code returned. Defaults to false. + /// Optional parameter that configures whether to get the inputs and outputs of the orchestration. Defaults to false. + /// A token that signals if the wait should be canceled. If canceled, call CreateCheckStatusResponseAsync to return a reponse contains a HttpManagementPayload. + /// + public static async Task WaitForCompletionOrCreateCheckStatusResponseAsync( + this DurableTaskClient client, + HttpRequestData request, + string instanceId, + TimeSpan? retryInterval = null, + bool returnInternalServerErrorOnFailure = false, + bool getInputsAndOutputs = false, + CancellationToken cancellation = default + ) + { + TimeSpan retryIntervalLocal = retryInterval ?? TimeSpan.FromSeconds(1); + try + { + while (true) + { + var status = await client.GetInstanceAsync(instanceId, getInputsAndOutputs: getInputsAndOutputs); + if (status != null) + { + if (status.RuntimeStatus == OrchestrationRuntimeStatus.Completed || +#pragma warning disable CS0618 // Type or member is obsolete + status.RuntimeStatus == OrchestrationRuntimeStatus.Canceled || +#pragma warning restore CS0618 // Type or member is obsolete + status.RuntimeStatus == OrchestrationRuntimeStatus.Terminated || + status.RuntimeStatus == OrchestrationRuntimeStatus.Failed) + { + var response = request.CreateResponse( + (status.RuntimeStatus == OrchestrationRuntimeStatus.Failed && returnInternalServerErrorOnFailure) ? HttpStatusCode.InternalServerError : HttpStatusCode.OK); + await response.WriteAsJsonAsync(new + { + Name = status.Name, + InstanceId = status.InstanceId, + CreatedAt = status.CreatedAt, + LastUpdatedAt = status.LastUpdatedAt, + RuntimeStatus = status.RuntimeStatus.ToString(), // Convert enum to string + SerializedInput = status.SerializedInput, + SerializedOutput = status.SerializedOutput, + SerializedCustomStatus = status.SerializedCustomStatus + }, statusCode: response.StatusCode); + + return response; + } + } + await Task.Delay(retryIntervalLocal, cancellation); + } + } + // If the task is canceled, call CreateCheckStatusResponseAsync to return a response containing instance management URLs. + catch (OperationCanceledException) + { + return await CreateCheckStatusResponseAsync(client, request, instanceId); + } + } + /// /// Creates an HTTP response that is useful for checking the status of the specified instance. /// @@ -120,8 +185,30 @@ public static HttpResponseData CreateCheckStatusResponse( return response; } - private static object SetHeadersAndGetPayload( - DurableTaskClient client, HttpRequestData request, HttpResponseData response, string instanceId) + /// + /// Creates an HTTP management payload for the specified orchestration instance. + /// + /// The . + /// The ID of the orchestration instance. + /// Optional HTTP request data to use for creating the base URL. + /// An object containing instance control URLs. + /// Thrown when instanceId is null or empty. + /// Thrown when a valid base URL cannot be determined. + public static HttpManagementPayload CreateHttpManagementPayload( + this DurableTaskClient client, + string instanceId, + HttpRequestData? request = null) + { + if (string.IsNullOrEmpty(instanceId)) + { + throw new ArgumentException("InstanceId cannot be null or empty.", nameof(instanceId)); + } + + return SetHeadersAndGetPayload(client, request, null, instanceId); + } + + private static HttpManagementPayload SetHeadersAndGetPayload( + DurableTaskClient client, HttpRequestData? request, HttpResponseData? response, string instanceId) { static string BuildUrl(string url, params string?[] queryValues) { @@ -143,22 +230,46 @@ static string BuildUrl(string url, params string?[] queryValues) // request headers into consideration and generate the base URL accordingly. // More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded. // One potential workaround is to set ASPNETCORE_FORWARDEDHEADERS_ENABLED to true. - string baseUrl = request.Url.GetLeftPart(UriPartial.Authority); + + // If HttpRequestData is provided, use its URL; otherwise, get the baseUrl from the DurableTaskClient. + // The base URL could be null if: + // 1. The DurableTaskClient isn't a FunctionsDurableTaskClient (which would have the baseUrl from bindings) + // 2. There's no valid HttpRequestData provided + string? baseUrl = ((request != null) ? GetBaseUrlFromRequest(request) : GetBaseUrl(client)); + + if (baseUrl == null) + { + throw new InvalidOperationException("Failed to create HTTP management payload as base URL is null. Either use Functions bindings or provide an HTTP request to create the HttpPayload."); + } + + bool isFromRequest = request != null; + string formattedInstanceId = Uri.EscapeDataString(instanceId); - string instanceUrl = $"{baseUrl}/runtime/webhooks/durabletask/instances/{formattedInstanceId}"; + + // The baseUrl differs depending on the source. Eg: + // - From request: http://localhost:7071/ + // - From durable client: http://localhost:7071/runtime/webhooks/durabletask + // We adjust the instanceUrl construction accordingly. + string instanceUrl = isFromRequest + ? $"{baseUrl}/runtime/webhooks/durabletask/instances/{formattedInstanceId}" + : $"{baseUrl}/instances/{formattedInstanceId}"; string? commonQueryParameters = GetQueryParams(client); - response.Headers.Add("Location", BuildUrl(instanceUrl, commonQueryParameters)); - response.Headers.Add("Content-Type", "application/json"); + + if (response != null) + { + response.Headers.Add("Location", BuildUrl(instanceUrl, commonQueryParameters)); + response.Headers.Add("Content-Type", "application/json"); + } - return new + return new HttpManagementPayload { - id = instanceId, - purgeHistoryDeleteUri = BuildUrl(instanceUrl, commonQueryParameters), - sendEventPostUri = BuildUrl($"{instanceUrl}/raiseEvent/{{eventName}}", commonQueryParameters), - statusQueryGetUri = BuildUrl(instanceUrl, commonQueryParameters), - terminatePostUri = BuildUrl($"{instanceUrl}/terminate", "reason={{text}}", commonQueryParameters), - suspendPostUri = BuildUrl($"{instanceUrl}/suspend", "reason={{text}}", commonQueryParameters), - resumePostUri = BuildUrl($"{instanceUrl}/resume", "reason={{text}}", commonQueryParameters) + Id = instanceId, + PurgeHistoryDeleteUri = BuildUrl(instanceUrl, commonQueryParameters), + SendEventPostUri = BuildUrl($"{instanceUrl}/raiseEvent/{{eventName}}", commonQueryParameters), + StatusQueryGetUri = BuildUrl(instanceUrl, commonQueryParameters), + TerminatePostUri = BuildUrl($"{instanceUrl}/terminate", "reason={{text}}", commonQueryParameters), + SuspendPostUri = BuildUrl($"{instanceUrl}/suspend", "reason={{text}}", commonQueryParameters), + ResumePostUri = BuildUrl($"{instanceUrl}/resume", "reason={{text}}", commonQueryParameters) }; } @@ -168,8 +279,58 @@ private static ObjectSerializer GetObjectSerializer(HttpResponseData response) ?? throw new InvalidOperationException("A serializer is not configured for the worker."); } + private static string? GetBaseUrlFromRequest(HttpRequestData request) + { + // Default to the scheme from the request URL + string proto = request.Url.Scheme; + string host = request.Url.Authority; + + // Check for "Forwarded" header + if (request.Headers.TryGetValues("Forwarded", out var forwardedHeaders)) + { + var forwardedDict = forwardedHeaders.FirstOrDefault()?.Split(';') + .Select(pair => pair.Split('=')) + .Where(pair => pair.Length == 2) + .ToDictionary(pair => pair[0].Trim(), pair => pair[1].Trim()); + + if (forwardedDict != null) + { + if (forwardedDict.TryGetValue("proto", out var forwardedProto)) + { + proto = forwardedProto; + } + if (forwardedDict.TryGetValue("host", out var forwardedHost)) + { + host = forwardedHost; + // Return if either proto or host (or both) were found in "Forwarded" header + return $"{proto}://{forwardedHost}"; + } + } + } + // Check for "X-Forwarded-Proto" and "X-Forwarded-Host" headers if "Forwarded" is not present + if (request.Headers.TryGetValues("X-Forwarded-Proto", out var protos)) + { + proto = protos.FirstOrDefault() ?? proto; + } + if (request.Headers.TryGetValues("X-Forwarded-Host", out var hosts)) + { + // Return base URL if either "X-Forwarded-Proto" or "X-Forwarded-Host" (or both) are found + host = hosts.FirstOrDefault() ?? host; + return $"{proto}://{host}"; + } + + // Construct and return the base URL from default fallback values + return $"{proto}://{host}"; + } + + private static string? GetQueryParams(DurableTaskClient client) { return client is FunctionsDurableTaskClient functions ? functions.QueryString : null; } + + private static string? GetBaseUrl(DurableTaskClient client) + { + return client is FunctionsDurableTaskClient functions ? functions.HttpBaseUrl : null; + } } diff --git a/src/Worker.Extensions.DurableTask/DurableTaskExtensionStartup.cs b/src/Worker.Extensions.DurableTask/DurableTaskExtensionStartup.cs index 626acd6bf..af7bab017 100644 --- a/src/Worker.Extensions.DurableTask/DurableTaskExtensionStartup.cs +++ b/src/Worker.Extensions.DurableTask/DurableTaskExtensionStartup.cs @@ -1,20 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; -using Azure.Core.Serialization; using Microsoft.Azure.Functions.Worker.Core; using Microsoft.Azure.Functions.Worker.Extensions.DurableTask; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Converters; -using Microsoft.DurableTask.Worker; -using Microsoft.DurableTask.Worker.Shims; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; [assembly: WorkerExtensionStartup(typeof(DurableTaskExtensionStartup))] @@ -28,49 +16,6 @@ public sealed class DurableTaskExtensionStartup : WorkerExtensionStartup /// public override void Configure(IFunctionsWorkerApplicationBuilder applicationBuilder) { - applicationBuilder.Services.AddSingleton(); - applicationBuilder.Services.AddOptions() - .Configure(options => options.EnableEntitySupport = true) - .PostConfigure((opt, sp) => - { - if (GetConverter(sp) is DataConverter converter) - { - opt.DataConverter = converter; - } - }); - - applicationBuilder.Services.AddOptions() - .Configure(options => options.EnableEntitySupport = true) - .PostConfigure((opt, sp) => - { - if (GetConverter(sp) is DataConverter converter) - { - opt.DataConverter = converter; - } - }); - - applicationBuilder.Services.TryAddSingleton(sp => - { - DurableTaskWorkerOptions options = sp.GetRequiredService>().Value; - ILoggerFactory factory = sp.GetRequiredService(); - return new DurableTaskShimFactory(options, factory); // For GrpcOrchestrationRunner - }); - - applicationBuilder.Services.Configure(o => - { - o.InputConverters.Register(); - }); - - applicationBuilder.UseMiddleware(); - } - - private static DataConverter? GetConverter(IServiceProvider services) - { - // We intentionally do not consider a DataConverter in the DI provider, or if one was already set. This is to - // ensure serialization is consistent with the rest of Azure Functions. This is particularly important because - // TaskActivity bindings use ObjectSerializer directly for the time being. Due to this, allowing DataConverter - // to be set separately from ObjectSerializer would give an inconsistent serialization solution. - WorkerOptions? worker = services.GetRequiredService>()?.Value; - return worker?.Serializer is not null ? new ObjectConverterShim(worker.Serializer) : null; + applicationBuilder.ConfigureDurableExtension(); } } diff --git a/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs b/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs index 0f9231375..3c919362d 100644 --- a/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs +++ b/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs @@ -17,15 +17,16 @@ internal sealed class FunctionsDurableTaskClient : DurableTaskClient { private readonly DurableTaskClient inner; - public FunctionsDurableTaskClient(DurableTaskClient inner, string? queryString) + public FunctionsDurableTaskClient(DurableTaskClient inner, string? queryString, string? httpBaseUrl) : base(inner.Name) { this.inner = inner; this.QueryString = queryString; + this.HttpBaseUrl = httpBaseUrl; } public string? QueryString { get; } - + public string? HttpBaseUrl { get; } public override DurableEntityClient Entities => this.inner.Entities; public override ValueTask DisposeAsync() diff --git a/src/Worker.Extensions.DurableTask/FunctionsWorkerApplicationBuilderExtensions.cs b/src/Worker.Extensions.DurableTask/FunctionsWorkerApplicationBuilderExtensions.cs new file mode 100644 index 000000000..642446dd4 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/FunctionsWorkerApplicationBuilderExtensions.cs @@ -0,0 +1,125 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Azure.Core.Serialization; +using Microsoft.Azure.Functions.Worker.Core; +using Microsoft.Azure.Functions.Worker.Extensions.DurableTask; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Shims; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.Functions.Worker; + +/// +/// Extensions for . +/// +public static class FunctionsWorkerApplicationBuilderExtensions +{ + /// + /// Configures the Durable Functions extension for the worker. + /// + /// The builder to configure. + /// The for call chaining. + public static IFunctionsWorkerApplicationBuilder ConfigureDurableExtension(this IFunctionsWorkerApplicationBuilder builder) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + + builder.Services.TryAddSingleton(); + builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton, ConfigureClientOptions>()); + builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton, PostConfigureClientOptions>()); + builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton, ConfigureWorkerOptions>()); + builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton, PostConfigureWorkerOptions>()); + + builder.Services.TryAddSingleton(sp => + { + DurableTaskWorkerOptions options = sp.GetRequiredService>().Value; + ILoggerFactory factory = sp.GetRequiredService(); + return new DurableTaskShimFactory(options, factory); // For GrpcOrchestrationRunner + }); + + builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton, ConfigureInputConverter>()); + if (!builder.Services.Any(d => d.ServiceType == typeof(DurableTaskFunctionsMiddleware))) + { + builder.UseMiddleware(); + } + + return builder; + } + + private class ConfigureInputConverter : IConfigureOptions + { + public void Configure(WorkerOptions options) + { + options.InputConverters.Register(); + } + } + + private class ConfigureClientOptions : IConfigureOptions + { + public void Configure(DurableTaskClientOptions options) + { + options.EnableEntitySupport = true; + } + } + + private class PostConfigureClientOptions : IPostConfigureOptions + { + readonly IOptionsMonitor workerOptions; + + public PostConfigureClientOptions(IOptionsMonitor workerOptions) + { + this.workerOptions = workerOptions; + } + + public void PostConfigure(string name, DurableTaskClientOptions options) + { + if (this.workerOptions.Get(name).Serializer is { } serializer) + { + options.DataConverter = new ObjectConverterShim(serializer); + } + } + } + + private class ConfigureWorkerOptions : IConfigureOptions + { + public void Configure(DurableTaskWorkerOptions options) + { + options.EnableEntitySupport = true; + } + } + + private class PostConfigureWorkerOptions : IPostConfigureOptions + { + readonly IOptionsMonitor workerOptions; + + public PostConfigureWorkerOptions(IOptionsMonitor workerOptions) + { + this.workerOptions = workerOptions; + } + + public void PostConfigure(string name, DurableTaskWorkerOptions options) + { + if (this.workerOptions.Get(name).Serializer is { } serializer) + { + options.DataConverter = new ObjectConverterShim(serializer); + } + } + } +} diff --git a/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs b/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs new file mode 100644 index 000000000..4e7228f87 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs @@ -0,0 +1,97 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// This is a copy of: https://github.com/Azure/azure-functions-durable-extension/blob/dev/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs + +using System; +using System.Collections.Generic; +using System.Text; +using Newtonsoft.Json; + +namespace Microsoft.Azure.Functions.Worker; + +/// +/// Data structure containing status, terminate and send external event HTTP endpoints. +/// +public class HttpManagementPayload +{ + /// + /// Gets the ID of the orchestration instance. + /// + /// + /// The ID of the orchestration instance. + /// + [JsonProperty("id")] + public string? Id { get; internal set; } + + /// + /// Gets the HTTP GET status query endpoint URL. + /// + /// + /// The HTTP URL for fetching the instance status. + /// + [JsonProperty("statusQueryGetUri")] + public string? StatusQueryGetUri { get; internal set; } + + /// + /// Gets the HTTP POST external event sending endpoint URL. + /// + /// + /// The HTTP URL for posting external event notifications. + /// + [JsonProperty("sendEventPostUri")] + public string? SendEventPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance termination endpoint. + /// + /// + /// The HTTP URL for posting instance termination commands. + /// + [JsonProperty("terminatePostUri")] + public string? TerminatePostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance rewind endpoint. + /// + /// + /// The HTTP URL for rewinding orchestration instances. + /// + [JsonProperty("rewindPostUri")] + public string? RewindPostUri { get; internal set; } + + /// + /// Gets the HTTP DELETE purge instance history by instance ID endpoint. + /// + /// + /// The HTTP URL for purging instance history by instance ID. + /// + [JsonProperty("purgeHistoryDeleteUri")] + public string? PurgeHistoryDeleteUri { get; internal set; } + + /// + /// Gets the HTTP POST instance restart endpoint. + /// + /// + /// The HTTP URL for restarting an orchestration instance. + /// + [JsonProperty("restartPostUri")] + public string? RestartPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance suspend endpoint. + /// + /// + /// The HTTP URL for suspending an orchestration instance. + /// + [JsonProperty("suspendPostUri")] + public string? SuspendPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance resume endpoint. + /// + /// + /// The HTTP URL for resuming an orchestration instance. + /// + [JsonProperty("resumePostUri")] + public string? ResumePostUri { get; internal set; } +} diff --git a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj index 7a93ae655..0276b570b 100644 --- a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj +++ b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj @@ -29,7 +29,7 @@ ..\..\sign.snk - 1.1.3 + 1.1.7 $(VersionPrefix).0 @@ -39,12 +39,13 @@ - - + + + diff --git a/test/Common/AzureStorageAccountProviderTests.cs b/test/Common/AzureStorageAccountProviderTests.cs index 67774b2ec..886903f37 100644 --- a/test/Common/AzureStorageAccountProviderTests.cs +++ b/test/Common/AzureStorageAccountProviderTests.cs @@ -80,11 +80,11 @@ public void GetStorageAccountDetails_ConfigSection_Endpoints() Assert.Equal(options.TableServiceUri, actual.TableServiceUri); // Get CloudStorageAccount - CloudStorageAccount acount = actual.ToCloudStorageAccount(); - Assert.Same(actual.StorageCredentials, acount.Credentials); - Assert.Equal(options.BlobServiceUri, acount.BlobEndpoint); - Assert.Equal(options.QueueServiceUri, acount.QueueEndpoint); - Assert.Equal(options.TableServiceUri, acount.TableEndpoint); + CloudStorageAccount account = actual.ToCloudStorageAccount(); + Assert.Same(actual.StorageCredentials, account.Credentials); + Assert.Equal(options.BlobServiceUri, account.BlobEndpoint); + Assert.Equal(options.QueueServiceUri, account.QueueEndpoint); + Assert.Equal(options.TableServiceUri, account.TableEndpoint); } [Fact] diff --git a/test/Common/CustomTestStorageAccountProvider.cs b/test/Common/CustomTestStorageAccountProvider.cs new file mode 100644 index 000000000..41337555a --- /dev/null +++ b/test/Common/CustomTestStorageAccountProvider.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using DurableTask.AzureStorage; +using Microsoft.WindowsAzure.Storage; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests +{ + internal class CustomTestStorageAccountProvider : IStorageAccountProvider + { + private readonly string customConnectionString; + private readonly string customConnectionName; + + public CustomTestStorageAccountProvider(string connectionName) + { + this.customConnectionName = connectionName; + this.customConnectionString = $"DefaultEndpointsProtocol=https;AccountName=test;AccountKey={GenerateRandomKey()};EndpointSuffix=core.windows.net"; + } + + public CloudStorageAccount GetCloudStorageAccount(string name) => + CloudStorageAccount.Parse(name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString); + + public StorageAccountDetails GetStorageAccountDetails(string name) => + new StorageAccountDetails { ConnectionString = name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString }; + + private static string GenerateRandomKey() + { + string key = Guid.NewGuid().ToString(); + return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(key)); + } + } +} diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 1044eddab..0ba61d7f3 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -746,6 +746,9 @@ await TestHelpers.WaitUntilTrue( conditionDescription: "Log file exists", timeout: TimeSpan.FromSeconds(30)); + // add a minute wait to ensure logs are fully written + await Task.Delay(TimeSpan.FromMinutes(1)); + await TestHelpers.WaitUntilTrue( predicate: () => { @@ -4299,7 +4302,7 @@ public async Task DurableEntity_EntityProxy_NameResolve(bool extendedSessions) } /// - /// Test which validates that entity state deserialization + /// Test which validates that entity state deserialization. /// [Theory] [InlineData(true)] @@ -5062,6 +5065,8 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) var orchestrationA = $"{prefix}-A"; var orchestrationB = $"{prefix}-B"; + // PART 1: Test removal of empty entities + // create an empty entity var client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.CreateEmptyEntities), new EntityId[] { emptyEntityId }, this.output); var status = await client.WaitForCompletionAsync(this.output); @@ -5081,6 +5086,17 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) var result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); Assert.Contains(result.Entities, s => s.EntityId.Equals(emptyEntityId)); + // test removal of empty entity + var response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: false, CancellationToken.None); + Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); + Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); + + // check that the empty entity record has been removed from storage + result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); + Assert.DoesNotContain(result.Entities, s => s.EntityId.Equals(emptyEntityId)); + + // PART 2: Test recovery from orphaned locks + // run an orchestration A that leaves an orphaned lock TestDurableClient clientA = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, true), this.output, orchestrationA); status = await clientA.WaitForCompletionAsync(this.output); @@ -5088,21 +5104,20 @@ public async Task DurableEntity_CleanEntityStorage(string storageProvider) // run an orchestration B that queues behind A for the lock (and thus gets stuck) TestDurableClient clientB = await host.StartOrchestratorAsync(nameof(TestOrchestrations.LockThenFailReplay), (orphanedEntityId, false), this.output, orchestrationB); - // remove empty entity and release orphaned lock - var response = await client.InnerClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + await Task.Delay(TimeSpan.FromMinutes(1)); // wait for a stable entity executionID, needed until https://github.com/Azure/durabletask/pull/1128 is merged + + // remove release orphaned lock to unblock orchestration B + // Note: do NOT remove empty entities yet: we want to keep the empty entity so it can unblock orchestration B + response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: false, releaseOrphanedLocks: true, CancellationToken.None); Assert.Equal(1, response.NumberOfOrphanedLocksRemoved); - Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); + Assert.Equal(0, response.NumberOfEmptyEntitiesRemoved); // wait for orchestration B to complete, now that the lock has been released status = await clientB.WaitForCompletionAsync(this.output); Assert.True(status.RuntimeStatus == OrchestrationRuntimeStatus.Completed); - // check that the empty entity record has been removed from storage - result = await client.InnerClient.ListEntitiesAsync(query, CancellationToken.None); - Assert.DoesNotContain(result.Entities, s => s.EntityId.Equals(emptyEntityId)); - // clean again to remove the orphaned entity which is now empty also - response = await client.InnerClient.CleanEntityStorageAsync(true, true, CancellationToken.None); + response = await client.InnerClient.CleanEntityStorageAsync(removeEmptyEntities: true, releaseOrphanedLocks: true, CancellationToken.None); Assert.Equal(0, response.NumberOfOrphanedLocksRemoved); Assert.Equal(1, response.NumberOfEmptyEntitiesRemoved); @@ -5577,16 +5592,24 @@ public async Task ExtendedSessions_OutOfProc_SetToFalse() { "FUNCTIONS_WORKER_RUNTIME", "node" }, }); - using (var host = TestHelpers.GetJobHostWithOptions( - this.loggerProvider, - durableTaskOptions, - nameResolver: nameResolver)) - { - await host.StartAsync(); - await host.StopAsync(); - } + InvalidOperationException exception = + await Assert.ThrowsAsync(async () => + { + using (var host = TestHelpers.GetJobHostWithOptions( + this.loggerProvider, + durableTaskOptions, + nameResolver: nameResolver)) + { + await host.StartAsync(); + await host.StopAsync(); + } + }); - Assert.False(durableTaskOptions.ExtendedSessionsEnabled); + Assert.NotNull(exception); + Assert.StartsWith( + "Durable Functions with extendedSessionsEnabled set to 'true' is only supported when using", + exception.Message, + StringComparison.OrdinalIgnoreCase); } [Fact] diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index be254329e..615d20874 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -41,7 +41,9 @@ public void CreateCheckStatusResponse_Throws_Exception_When_NotificationUrl_Miss var httpApiHandler = new HttpApiHandler(GetTestExtension(options), null); var ex = Assert.Throws(() => httpApiHandler.CreateCheckStatusResponse(new HttpRequestMessage(), string.Empty, null)); - Assert.Equal("Webhooks are not configured", ex.Message); + string errorMessage = "Webhooks are not configured. This may occur if the environment variable `WEBSITE_HOSTNAME` is not set (should be automatically set for Azure Functions). " + + "Try setting it to the appropiate URI to reach your app. For example: the DNS name of the app, or a value of the form :."; + Assert.Equal(errorMessage, ex.Message); } [Fact] @@ -409,14 +411,14 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP TaskHub = TestConstants.TaskHub, ConnectionName = TestConstants.ConnectionName, }, - TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(3)); stopwatch.Stop(); Assert.Equal(HttpStatusCode.OK, httpResponseMessage.StatusCode); var content = await httpResponseMessage.Content.ReadAsStringAsync(); var value = JsonConvert.DeserializeObject(content); Assert.Equal("Hello Tokyo!", value); - Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(10)); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(15)); } [Fact] diff --git a/test/Common/TestDurableClient.cs b/test/Common/TestDurableClient.cs index d9f41d4c4..6e35e5e49 100644 --- a/test/Common/TestDurableClient.cs +++ b/test/Common/TestDurableClient.cs @@ -182,7 +182,7 @@ public async Task WaitForCompletionAsync( { if (timeout == null) { - timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(30); + timeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromMinutes(1); } Stopwatch sw = Stopwatch.StartNew(); diff --git a/test/Common/TestHelpers.cs b/test/Common/TestHelpers.cs index 83259db08..a6c455e3a 100644 --- a/test/Common/TestHelpers.cs +++ b/test/Common/TestHelpers.cs @@ -12,8 +12,8 @@ using DurableTask.AzureStorage; using Microsoft.ApplicationInsights.Channel; #if !FUNCTIONS_V1 -using Microsoft.Extensions.Hosting; using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Extensions.Hosting; #endif using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.Extensions.Logging; @@ -703,7 +703,7 @@ private static List GetLogs_UnhandledOrchestrationException(string messa var list = new List() { $"{messageId}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' scheduled. Reason: NewInstance. IsReplay: False.", - $"{messageId}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' started. IsReplay: False. Input: (null)", + $"{messageId}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' started. IsReplay: False. Input: ", $"{messageId}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' failed with an error. Reason: System.ArgumentNullException: Value cannot be null.", }; @@ -831,7 +831,7 @@ private static List GetLogs_Orchestration_Activity(string[] messageIds, $"{messageIds[1]}:0: Function '{orchestratorFunctionNames[1]} ({FunctionType.Orchestrator})' completed. ContinuedAsNew: False. IsReplay: False. Output: \"Hello,", $"{messageIds[0]}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' started. IsReplay: True.", $"{messageIds[0]}: Function '{orchestratorFunctionNames[1]} ({FunctionType.Orchestrator})' scheduled. Reason: OrchestratorGreeting. IsReplay: True.", - $"{messageIds[0]}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' completed. ContinuedAsNew: False. IsReplay: False. Output: (null)", + $"{messageIds[0]}: Function '{orchestratorFunctionNames[0]} ({FunctionType.Orchestrator})' completed. ContinuedAsNew: False. IsReplay: False. Output: ", }; return list; diff --git a/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs b/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs index 30363c325..a55933384 100644 --- a/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs +++ b/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs @@ -154,5 +154,46 @@ public void EnvironmentIsVMSS_WorkerIdFromEnvironmentVariables() Assert.Equal("waws-prod-euapbn1-003:dw0SmallDedicatedWebWorkerRole_hr0HostRole-3-VM-13", settings.WorkerId); } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void CustomConnectionNameIsResolved() + { + var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection"); + var mockOptions = new OptionsWrapper(new DurableTaskOptions()); + var nameResolver = new Mock().Object; + + var factory = new AzureStorageDurabilityProviderFactory( + mockOptions, + storageAccountProvider, + nameResolver, + NullLoggerFactory.Instance, + TestHelpers.GetMockPlatformInformationService()); + + factory.GetDurabilityProvider(); // This will initialize the default connection string + var provider = factory.GetDurabilityProvider(new DurableClientAttribute() { ConnectionName = "CustomConnection", TaskHub = "TestHubName" }); + + Assert.Equal("CustomConnection", provider.ConnectionName); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void DefaultConnectionNameIsResolved() + { + var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection"); + var mockOptions = new OptionsWrapper(new DurableTaskOptions()); + var nameResolver = new Mock().Object; + + var factory = new AzureStorageDurabilityProviderFactory( + mockOptions, + storageAccountProvider, + nameResolver, + NullLoggerFactory.Instance, + TestHelpers.GetMockPlatformInformationService()); + + var provider = factory.GetDurabilityProvider(); + + Assert.Equal("Storage", provider.ConnectionName); + } } } diff --git a/test/FunctionsV2/CorrelationEndToEndTests.cs b/test/FunctionsV2/CorrelationEndToEndTests.cs index 0b9e31eac..6eb2e9090 100644 --- a/test/FunctionsV2/CorrelationEndToEndTests.cs +++ b/test/FunctionsV2/CorrelationEndToEndTests.cs @@ -234,7 +234,7 @@ internal async Task, List>> [InlineData(false, true, true)] [InlineData(true, true, false)] [InlineData(true, true, true)] - public async void TelemetryClientSetup_AppInsights_Warnings(bool instrumentationKeyIsSet, bool connStringIsSet, bool extendedSessions) + public void TelemetryClientSetup_AppInsights_Warnings(bool instrumentationKeyIsSet, bool connStringIsSet, bool extendedSessions) { TraceOptions traceOptions = new TraceOptions() { @@ -258,11 +258,11 @@ public async void TelemetryClientSetup_AppInsights_Warnings(bool instrumentation } else if (instrumentationKeyIsSet) { - mockNameResolver = GetNameResolverMock(new[] { (instKeyEnvVarName, environmentVariableValue), (connStringEnvVarName, String.Empty) }); + mockNameResolver = GetNameResolverMock(new[] { (instKeyEnvVarName, environmentVariableValue), (connStringEnvVarName, string.Empty) }); } else if (connStringIsSet) { - mockNameResolver = GetNameResolverMock(new[] { (instKeyEnvVarName, String.Empty), (connStringEnvVarName, connStringValue) }); + mockNameResolver = GetNameResolverMock(new[] { (instKeyEnvVarName, string.Empty), (connStringEnvVarName, connStringValue) }); } using (var host = TestHelpers.GetJobHost( @@ -405,14 +405,6 @@ private static List GetCorrelationSortedList(OperationTeleme var result = new List(); if (current.Count != 0) { - foreach (var some in current) - { - if (parent.Id == some.Context.Operation.ParentId) - { - Console.WriteLine("match"); - } - } - IOrderedEnumerable nexts = current.Where(p => p.Context.Operation.ParentId == parent.Id).OrderBy(p => p.Timestamp.Ticks); foreach (OperationTelemetry next in nexts) { diff --git a/test/FunctionsV2/EndToEndTraceHelperTests.cs b/test/FunctionsV2/EndToEndTraceHelperTests.cs new file mode 100644 index 000000000..4ea1b4e06 --- /dev/null +++ b/test/FunctionsV2/EndToEndTraceHelperTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#nullable enable +using System; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace WebJobs.Extensions.DurableTask.Tests.V2 +{ + public class EndToEndTraceHelperTests + { + [Theory] + [InlineData(true, "DO NOT LOG ME")] + [InlineData(false, "DO NOT LOG ME")] + [InlineData(true, null)] + [InlineData(false, null)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void StringSanitizerTest( + bool shouldTraceRawData, + string? possiblySensitiveData) + { + // set up trace helper + var nullLogger = new NullLogger(); + var traceHelper = new EndToEndTraceHelper( + logger: nullLogger, + traceReplayEvents: false, // has not effect on sanitizer + shouldTraceRawData: shouldTraceRawData); + + // run sanitizer + traceHelper.SanitizeString( + rawPayload: possiblySensitiveData, + out string iLoggerString, + out string kustoTableString); + + // expected: sanitized string should not contain the sensitive data + // skip this check if data is null + if (possiblySensitiveData != null) + { + Assert.DoesNotContain(possiblySensitiveData, kustoTableString); + } + + if (shouldTraceRawData) + { + string expectedString = possiblySensitiveData ?? string.Empty; + Assert.Equal(expectedString, iLoggerString); + } + else + { + // If raw data is not being traced, + // kusto and the ilogger should get the same data + Assert.Equal(iLoggerString, kustoTableString); + } + } + + [Theory] + [InlineData(true, "DO NOT LOG ME")] + [InlineData(false, "DO NOT LOG ME")] + [InlineData(true, null)] + [InlineData(false, null)] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void ExceptionSanitizerTest( + bool shouldTraceRawData, + string? possiblySensitiveData) + { + // set up trace helper + var nullLogger = new NullLogger(); + var traceHelper = new EndToEndTraceHelper( + logger: nullLogger, + traceReplayEvents: false, // has not effect on sanitizer + shouldTraceRawData: shouldTraceRawData); + + // exception to sanitize + Exception? exception = null; + if (possiblySensitiveData != null) + { + exception = new Exception(possiblySensitiveData); + } + + // run sanitizer + traceHelper.SanitizeException( + exception: exception, + out string iLoggerString, + out string kustoTableString); + + // exception message should not be part of the sanitized strings + // skip this check if data is null + if (possiblySensitiveData != null) + { + Assert.DoesNotContain(possiblySensitiveData, kustoTableString); + } + + if (shouldTraceRawData) + { + var expectedString = exception?.ToString() ?? string.Empty; + Assert.Equal(expectedString, iLoggerString); + } + else + { + // If raw data is not being traced, + // kusto and the ilogger should get the same data + Assert.Equal(iLoggerString, kustoTableString); + } + } + } +} diff --git a/test/FunctionsV2/OutOfProcTests.cs b/test/FunctionsV2/OutOfProcTests.cs index 4f95c1a2b..69fa450a5 100644 --- a/test/FunctionsV2/OutOfProcTests.cs +++ b/test/FunctionsV2/OutOfProcTests.cs @@ -342,6 +342,7 @@ public async Task TestLocalRcpEndpointRuntimeVersion(string runtimeVersion, bool // Validate if we opened local RPC endpoint by looking at log statements. var logger = this.loggerProvider.CreatedLoggers.Single(l => l.Category == TestHelpers.LogCategory); var logMessages = logger.LogMessages.ToList(); + bool enabledRpcEndpoint = logMessages.Any(msg => msg.Level == Microsoft.Extensions.Logging.LogLevel.Information && msg.FormattedMessage.StartsWith($"Opened local {expectedProtocol} endpoint:")); Assert.Equal(enabledExpected, enabledRpcEndpoint); @@ -363,6 +364,7 @@ public async Task InvokeLocalRpcEndpoint() { await host.StartAsync(); +#pragma warning disable SYSLIB0014 // Type or member is obsolete using (var client = new WebClient()) { string jsonString = client.DownloadString("http://localhost:17071/durabletask/instances"); @@ -370,6 +372,7 @@ public async Task InvokeLocalRpcEndpoint() // The result is expected to be an empty array JArray array = JArray.Parse(jsonString); } +#pragma warning restore SYSLIB0014 // Type or member is obsolete await host.StopAsync(); } diff --git a/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs b/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs index 4b2d46456..428b93c9d 100644 --- a/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs +++ b/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs @@ -230,7 +230,6 @@ private static IWebJobsBuilder AddEmulatorDurableTask(this IWebJobsBuilder build internal class FunctionsV2HostWrapper : ITestHost { - internal readonly IHost InnerHost; private readonly JobHost innerWebJobsHost; private readonly DurableTaskOptions options; private readonly INameResolver nameResolver; @@ -255,6 +254,8 @@ internal FunctionsV2HostWrapper( this.options = options.Value; } + internal IHost InnerHost { get; private set; } + public Task CallAsync(string methodName, IDictionary args) => this.innerWebJobsHost.CallAsync(methodName, args); diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln new file mode 100644 index 000000000..a93cc6f6e --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetIsolated", "DotNetIsolated.csproj", "{B2DBA49D-9D25-46DB-8968-15D5E83B4060}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0954D7B4-582F-4F85-AE3E-5D503FB07DB1} + EndGlobalSection +EndGlobal diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs new file mode 100644 index 000000000..8332fa436 --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs @@ -0,0 +1,165 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using System; + +namespace FaultOrchestrators +{ + public static class FaultyOrchestrators + { + [Function(nameof(OOMOrchestrator))] + public static Task OOMOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force an OOM exception on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force the process to run out of memory + List data = new List(); + + for (int i = 0; i < 10000000; i++) + { + data.Add(new byte[1024 * 1024 * 1024]); + } + + // we expect the code to never reach this statement, it should OOM. + // we throw just in case the code does not time out. This should fail the test + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function(nameof(ProcessExitOrchestrator))] + public static Task ProcessExitOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force a sudden process exit on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force sudden crash + Environment.FailFast("Simulating crash!"); + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function(nameof(TimeoutOrchestrator))] + public static Task TimeoutOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force a timeout on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force the process to timeout after a 1 minute wait + System.Threading.Thread.Sleep(TimeSpan.FromMinutes(1)); + + // we expect the code to never reach this statement, it should time out. + // we throw just in case the code does not time out. This should fail the test + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function("durable_HttpStartOOMOrchestrator")] + public static async Task HttpStartOOMOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartOOMOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(OOMOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + + [Function("durable_HttpStartProcessExitOrchestrator")] + public static async Task HttpStartProcessExitOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartProcessExitOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(ProcessExitOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + + [Function("durable_HttpStartTimeoutOrchestrator")] + public static async Task HttpStartTimeoutOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartTimeoutOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(TimeoutOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + } +} diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json index 278b52cde..0ec9c6a89 100644 --- a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json @@ -7,5 +7,14 @@ "excludedTypes": "Request" } } - } + }, + "extensions": { + "durableTask": { + "storageProvider": { + "maxQueuePollingInterval": "00:00:01", + "controlQueueVisibilityTimeout": "00:01:00" + } + } + }, + "functionTimeout": "00:00:30" } \ No newline at end of file diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 new file mode 100644 index 000000000..79d679b80 --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 @@ -0,0 +1,119 @@ +# This is a simple test runner to validate the .NET isolated smoke tests. +# It supercedes the usual e2e-tests.ps1 script for the .NET isolated scenario because building the snmoke test app +# on the docker image is unreliable. For more details, see: https://github.com/Azure/azure-functions-host/issues/7995 + +# This script is designed specifically to test cases where the isolated worker process experiences a platform failure: +# timeouts, OOMs, etc. For that reason, it is careful to check that the Functions Host is running and healthy at regular +# intervals. This makes these tests run more slowly than other test categories. + +param( + [Parameter(Mandatory=$true)] + [string]$HttpStartPath +) + +$retryCount = 0; +$statusUrl = $null; +$success = $false; +$haveManuallyRestartedHost = $false; + +Do { + $testIsRunning = $true; + + # Start the functions host if it's not running already. + # Then give it up to 1 minute to start up. + # This is a long wait, but from experience the CI can be slow to start up the host, especially after a platform-error. + $isFunctionsHostRunning = (Get-Process -Name func -ErrorAction SilentlyContinue) + if ($isFunctionsHostRunning -eq $null) { + Write-Host "Starting the Functions host..." -ForegroundColor Yellow + + # The '&' operator is used to run the command in the background + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + Write-Host "Waiting for the Functions host to start up..." -ForegroundColor Yellow + Start-Sleep -Seconds 60 + } + + + try { + # Make sure the Functions runtime is up and running + $pingUrl = "http://localhost:7071/admin/host/ping" + Write-Host "Pinging app at $pingUrl to ensure the host is healthy" -ForegroundColor Yellow + Invoke-RestMethod -Method Post -Uri "http://localhost:7071/admin/host/ping" + Write-Host "Host is healthy!" -ForegroundColor Green + + # Start orchestrator if it hasn't been started yet + if ($statusUrl -eq $null){ + $startOrchestrationUri = "http://localhost:7071/$HttpStartPath" + Write-Host "Starting a new orchestration instance via POST to $startOrchestrationUri..." -ForegroundColor Yellow + + $result = Invoke-RestMethod -Method Post -Uri $startOrchestrationUri + Write-Host "Started orchestration with instance ID '$($result.id)'!" -ForegroundColor Yellow + Write-Host "Waiting for orchestration to complete..." -ForegroundColor Yellow + + $statusUrl = $result.statusQueryGetUri + + # sleep for a bit to give the orchestrator a chance to start, + # then loop once more in case the orchestrator ran quickly, made the host unhealthy, + # and the functions host needs to be restarted + Start-Sleep -Seconds 5 + continue; + } + + # Check the orchestrator status + $result = Invoke-RestMethod -Method Get -Uri $statusUrl + $runtimeStatus = $result.runtimeStatus + Write-Host "Orchestration is $runtimeStatus" -ForegroundColor Yellow + Write-Host $result + + if ($result.runtimeStatus -eq "Completed") { + $success = $true + $testIsRunning = $false + break + } + if ($result.runtimeStatus -eq "Failed") { + $success = $false + $testIsRunning = $false + break + } + + # If the orchestrator did not complete yet, wait for a bit before checking again + Start-Sleep -Seconds 2 + $retryCount = $retryCount + 1 + + } catch { + # we expect to enter this 'catch' block if any of our HTTP requests to the host fail. + # Some failures observed during development include: + # - The host is not running/was restarting/was killed + # - The host is running but not healthy (OOMs may cause this), so it needs to be forcibly restarted + Write-Host "An error occurred:" -ForegroundColor Red + Write-Host $_ -ForegroundColor Red + + # When testing for platform errors, we want to make sure the Functions host is healthy and ready to take requests. + # The Host can get into bad states (for example, in an OOM-inducing test) where it does not self-heal. + # For these cases, we manually restart the host to ensure it is in a good state. We only do this once per test. + if ($haveManuallyRestartedHost -eq $false) { + + # We stop the host process and wait for a bit before checking if it is running again. + Write-Host "Restarting the Functions host..." -ForegroundColor Yellow + Stop-Process -Name "func" -Force + Start-Sleep -Seconds 5 + + # Log whether the process kill succeeded + $haveManuallyRestartedHost = $true + $isFunctionsHostRunning = ((Get-Process -Name func -ErrorAction SilentlyContinue) -eq $null) + Write-Host "Host process killed: $isFunctionsHostRunning" -ForegroundColor Yellow + + # the beginning of the loop will restart the host + continue + } + + # Rethrow the original exception + throw + } + +} while (($testIsRunning -eq $true) -and ($retryCount -lt 65)) + +if ($success -eq $false) { + throw "Orchestration failed or did not compete in time! :(" +} + +Write-Host "Success!" -ForegroundColor Green \ No newline at end of file diff --git a/test/SmokeTests/OOProcSmokeTests/durableJS/Nuget.config b/test/SmokeTests/OOProcSmokeTests/durableJS/Nuget.config index c7e0e8535..4656064a6 100644 --- a/test/SmokeTests/OOProcSmokeTests/durableJS/Nuget.config +++ b/test/SmokeTests/OOProcSmokeTests/durableJS/Nuget.config @@ -1,8 +1,9 @@ + + - \ No newline at end of file diff --git a/test/SmokeTests/OOProcSmokeTests/durableJava/Nuget.config b/test/SmokeTests/OOProcSmokeTests/durableJava/Nuget.config index c7e0e8535..4656064a6 100644 --- a/test/SmokeTests/OOProcSmokeTests/durableJava/Nuget.config +++ b/test/SmokeTests/OOProcSmokeTests/durableJava/Nuget.config @@ -1,8 +1,9 @@ + + - \ No newline at end of file diff --git a/test/SmokeTests/OOProcSmokeTests/durablePy/Nuget.config b/test/SmokeTests/OOProcSmokeTests/durablePy/Nuget.config index c7e0e8535..4656064a6 100644 --- a/test/SmokeTests/OOProcSmokeTests/durablePy/Nuget.config +++ b/test/SmokeTests/OOProcSmokeTests/durablePy/Nuget.config @@ -1,8 +1,9 @@ + + - \ No newline at end of file diff --git a/test/SmokeTests/SmokeTestsV1/VSSampleV1.csproj b/test/SmokeTests/SmokeTestsV1/VSSampleV1.csproj index 1d2b30732..304a6fe3b 100644 --- a/test/SmokeTests/SmokeTestsV1/VSSampleV1.csproj +++ b/test/SmokeTests/SmokeTestsV1/VSSampleV1.csproj @@ -10,7 +10,6 @@ - diff --git a/test/SmokeTests/e2e-test.ps1 b/test/SmokeTests/e2e-test.ps1 index 845c35eb2..ab918da8e 100644 --- a/test/SmokeTests/e2e-test.ps1 +++ b/test/SmokeTests/e2e-test.ps1 @@ -26,12 +26,12 @@ function Exit-OnError() { } $ErrorActionPreference = "Stop" -$AzuriteVersion = "3.26.0" +$AzuriteVersion = "3.32.0" if ($NoSetup -eq $false) { # Build the docker image first, since that's the most critical step Write-Host "Building sample app Docker container from '$DockerfilePath'..." -ForegroundColor Yellow - docker build -f $DockerfilePath -t $ImageName --progress plain $PSScriptRoot/../../ + docker build --pull -f $DockerfilePath -t $ImageName --progress plain $PSScriptRoot/../../ Exit-OnError # Next, download and start the Azurite emulator Docker image @@ -58,6 +58,16 @@ if ($NoSetup -eq $false) { Start-Sleep -Seconds 30 # Adjust the sleep duration based on your SQL Server container startup time Exit-OnError + Write-Host "Checking if SQL Server is still running..." -ForegroundColor Yellow + $sqlServerStatus = docker inspect -f '{{.State.Status}}' mssql-server + Exit-OnError + + if ($sqlServerStatus -ne "running") { + Write-Host "Unexpected SQL Server status: $sqlServerStatus" -ForegroundColor Yellow + docker logs mssql-server + exit 1; + } + # Get SQL Server IP Address - used to create SQLDB_Connection Write-Host "Getting IP Address..." -ForegroundColor Yellow $serverIpAddress = docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mssql-server @@ -65,7 +75,7 @@ if ($NoSetup -eq $false) { # Create the database with strict binary collation Write-Host "Creating '$dbname' database with '$collation' collation" -ForegroundColor DarkYellow - docker exec -d mssql-server /opt/mssql-tools/bin/sqlcmd -S . -U sa -P "$pw" -Q "CREATE DATABASE [$dbname] COLLATE $collation" + docker exec -d mssql-server /opt/mssql-tools18/bin/sqlcmd -S . -U sa -P "$pw" -Q "CREATE DATABASE [$dbname] COLLATE $collation" Exit-OnError # Wait for database to be ready diff --git a/test/TimeoutTests/Python/Nuget.config b/test/TimeoutTests/Python/Nuget.config index c7e0e8535..4656064a6 100644 --- a/test/TimeoutTests/Python/Nuget.config +++ b/test/TimeoutTests/Python/Nuget.config @@ -1,8 +1,9 @@ + + - \ No newline at end of file diff --git a/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs b/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs index 5a335aefa..1623f4559 100644 --- a/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs +++ b/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs @@ -1,6 +1,10 @@ +using System.Net; +using Azure.Core.Serialization; +using Microsoft.Azure.Functions.Worker.Http; using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Client.Grpc; +using Microsoft.Extensions.Options; using Moq; +using Newtonsoft.Json; namespace Microsoft.Azure.Functions.Worker.Tests { @@ -9,7 +13,7 @@ namespace Microsoft.Azure.Functions.Worker.Tests /// public class FunctionsDurableTaskClientTests { - private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient() + private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient(string? baseUrl = null, OrchestrationMetadata? orchestrationMetadata = null) { // construct mock client @@ -21,8 +25,14 @@ private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient() durableClientMock.Setup(x => x.TerminateInstanceAsync( It.IsAny(), It.IsAny(), It.IsAny())).Returns(completedTask); + if (orchestrationMetadata != null) + { + durableClientMock.Setup(x => x.GetInstancesAsync(orchestrationMetadata.InstanceId, It.IsAny(), It.IsAny())) + .ReturnsAsync(orchestrationMetadata); + } + DurableTaskClient durableClient = durableClientMock.Object; - FunctionsDurableTaskClient client = new FunctionsDurableTaskClient(durableClient, queryString: null); + FunctionsDurableTaskClient client = new FunctionsDurableTaskClient(durableClient, queryString: null, httpBaseUrl: baseUrl); return client; } @@ -53,5 +63,274 @@ public async void TerminateDoesNotThrow() await client.TerminateInstanceAsync(instanceId, options); await client.TerminateInstanceAsync(instanceId, options, token); } + + /// + /// Test that the `CreateHttpManagementPayload` method returns the expected payload structure without HttpRequestData. + /// + [Fact] + public void CreateHttpManagementPayload_WithBaseUrl() + { + const string BaseUrl = "http://localhost:7071/runtime/webhooks/durabletask"; + FunctionsDurableTaskClient client = this.GetTestFunctionsDurableTaskClient(BaseUrl); + string instanceId = "testInstanceIdWithHostBaseUrl"; + + HttpManagementPayload payload = client.CreateHttpManagementPayload(instanceId); + + AssertHttpManagementPayload(payload, BaseUrl, instanceId); + } + + /// + /// Test that the `CreateHttpManagementPayload` method returns the expected payload structure with HttpRequestData. + /// + [Fact] + public void CreateHttpManagementPayload_WithHttpRequestData() + { + const string requestUrl = "http://localhost:7075/orchestrators/E1_HelloSequence"; + FunctionsDurableTaskClient client = this.GetTestFunctionsDurableTaskClient(); + string instanceId = "testInstanceIdWithRequest"; + + // Create mock HttpRequestData object. + var mockFunctionContext = new Mock(); + var mockHttpRequestData = new Mock(mockFunctionContext.Object); + var headers = new HttpHeadersCollection(); + mockHttpRequestData.SetupGet(r => r.Headers).Returns(headers); + mockHttpRequestData.SetupGet(r => r.Url).Returns(new Uri(requestUrl)); + + HttpManagementPayload payload = client.CreateHttpManagementPayload(instanceId, mockHttpRequestData.Object); + + AssertHttpManagementPayload(payload, "http://localhost:7075/runtime/webhooks/durabletask", instanceId); + } + + /// + /// Test that the `WaitForCompletionOrCreateCheckStatusResponseAsync` method returns the expected response when the orchestration is completed. + /// The expected response should include OrchestrationMetadata in the body with an HttpStatusCode.OK. + /// + [Fact] + public async Task TestWaitForCompletionOrCreateCheckStatusResponseAsync_WhenCompleted() + { + string instanceId = "test-instance-id-completed"; + var expectedResult = new OrchestrationMetadata("TestCompleted", instanceId) + { + CreatedAt = DateTime.UtcNow, + LastUpdatedAt = DateTime.UtcNow, + RuntimeStatus = OrchestrationRuntimeStatus.Completed, + SerializedCustomStatus = "TestCustomStatus", + SerializedInput = "TestInput", + SerializedOutput = "TestOutput" + }; + + var client = this.GetTestFunctionsDurableTaskClient( orchestrationMetadata: expectedResult); + + HttpRequestData request = this.MockHttpRequestAndResponseData(); + + HttpResponseData response = await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, instanceId); + + Assert.NotNull(response); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + // Reset stream position for reading + response.Body.Position = 0; + var orchestratorMetadata = await System.Text.Json.JsonSerializer.DeserializeAsync(response.Body); + + // Assert the response content is not null and check the content is correct. + Assert.NotNull(orchestratorMetadata); + AssertOrhcestrationMetadata(expectedResult, orchestratorMetadata); + } + + /// + /// Test that the `WaitForCompletionOrCreateCheckStatusResponseAsync` method returns expected response when the orchestrator didn't finish within + /// the timeout period. The response body should contain a HttpManagementPayload with HttpStatusCode.Accepted. + /// + [Fact] + public async Task TestWaitForCompletionOrCreateCheckStatusResponseAsync_WhenRunning() + { + string instanceId = "test-instance-id-running"; + var expectedResult = new OrchestrationMetadata("TestRunning", instanceId) + { + CreatedAt = DateTime.UtcNow, + LastUpdatedAt = DateTime.UtcNow, + RuntimeStatus = OrchestrationRuntimeStatus.Running, + }; + + var client = this.GetTestFunctionsDurableTaskClient(orchestrationMetadata: expectedResult); + + HttpRequestData request = this.MockHttpRequestAndResponseData(); + HttpResponseData response; + using (CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) + { + response = await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, instanceId, cancellation: cts.Token); + }; + + Assert.NotNull(response); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + // Reset stream position for reading + response.Body.Position = 0; + HttpManagementPayload? payload; + using (var reader = new StreamReader(response.Body)) + { + payload = JsonConvert.DeserializeObject(await reader.ReadToEndAsync()); + } + + // Assert the response content is not null and check the content is correct. + Assert.NotNull(payload); + AssertHttpManagementPayload(payload, "https://localhost:7075/runtime/webhooks/durabletask", instanceId); + } + + /// + /// Tests the `WaitForCompletionOrCreateCheckStatusResponseAsync` method to ensure it returns the correct HTTP status code + /// based on the `returnInternalServerErrorOnFailure` parameter when the orchestration has failed. + /// + [Theory] + [InlineData(true, HttpStatusCode.InternalServerError)] + [InlineData(false, HttpStatusCode.OK)] + public async Task TestWaitForCompletionOrCreateCheckStatusResponseAsync_WhenFailed(bool returnInternalServerErrorOnFailure, HttpStatusCode expected) + { + string instanceId = "test-instance-id-failed"; + var expectedResult = new OrchestrationMetadata("TestFailed", instanceId) + { + CreatedAt = DateTime.UtcNow, + LastUpdatedAt = DateTime.UtcNow, + RuntimeStatus = OrchestrationRuntimeStatus.Failed, + SerializedOutput = "Microsoft.DurableTask.TaskFailedException: Task 'SayHello' (#0) failed with an unhandled exception: Exception while executing function: Functions.SayHello", + SerializedInput = null + }; + + var client = this.GetTestFunctionsDurableTaskClient(orchestrationMetadata: expectedResult); + + HttpRequestData request = this.MockHttpRequestAndResponseData(); + + HttpResponseData response = await client.WaitForCompletionOrCreateCheckStatusResponseAsync(request, instanceId, returnInternalServerErrorOnFailure: returnInternalServerErrorOnFailure); + + Assert.NotNull(response); + Assert.Equal(expected, response.StatusCode); + + // Reset stream position for reading + response.Body.Position = 0; + var orchestratorMetadata = await System.Text.Json.JsonSerializer.DeserializeAsync(response.Body); + + // Assert the response content is not null and check the content is correct. + Assert.NotNull(orchestratorMetadata); + AssertOrhcestrationMetadata(expectedResult, orchestratorMetadata); + } + + /// + /// Tests the `GetBaseUrlFromRequest` can return the right base URL from the HttpRequestData with different forwarding or proxies. + /// This test covers the following scenarios: + /// - Using the "Forwarded" header + /// - Using "X-Forwarded-Proto" and "X-Forwarded-Host" headers + /// - Using only "X-Forwarded-Host" with default protocol + /// - no headers + /// + [Theory] + [InlineData("Forwarded", "proto=https;host=forwarded.example.com","","", "https://forwarded.example.com/runtime/webhooks/durabletask")] + [InlineData("X-Forwarded-Proto", "https", "X-Forwarded-Host", "xforwarded.example.com", "https://xforwarded.example.com/runtime/webhooks/durabletask")] + [InlineData("", "", "X-Forwarded-Host", "test.net", "https://test.net/runtime/webhooks/durabletask")] + [InlineData("", "", "", "", "https://localhost:7075/runtime/webhooks/durabletask")] // Default base URL for empty headers + public void TestHttpRequestDataForwardingHandling(string header1, string? value1, string header2, string value2, string expectedBaseUrl) + { + var headers = new HttpHeadersCollection(); + if (!string.IsNullOrEmpty(header1)) + { + headers.Add(header1, value1); + } + if (!string.IsNullOrEmpty(header2)) + { + headers.Add(header2, value2); + } + + var request = this.MockHttpRequestAndResponseData(headers); + var client = this.GetTestFunctionsDurableTaskClient(); + + var payload = client.CreateHttpManagementPayload("testInstanceId", request); + AssertHttpManagementPayload(payload, expectedBaseUrl, "testInstanceId"); + } + + + + private static void AssertHttpManagementPayload(HttpManagementPayload payload, string BaseUrl, string instanceId) + { + Assert.Equal(instanceId, payload.Id); + Assert.Equal($"{BaseUrl}/instances/{instanceId}", payload.PurgeHistoryDeleteUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/raiseEvent/{{eventName}}", payload.SendEventPostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}", payload.StatusQueryGetUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/terminate?reason={{{{text}}}}", payload.TerminatePostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/suspend?reason={{{{text}}}}", payload.SuspendPostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/resume?reason={{{{text}}}}", payload.ResumePostUri); + } + + private static void AssertOrhcestrationMetadata(OrchestrationMetadata expectedResult, dynamic actualResult) + { + Assert.Equal(expectedResult.Name, actualResult.GetProperty("Name").GetString()); + Assert.Equal(expectedResult.InstanceId, actualResult.GetProperty("InstanceId").GetString()); + Assert.Equal(expectedResult.CreatedAt, actualResult.GetProperty("CreatedAt").GetDateTime()); + Assert.Equal(expectedResult.LastUpdatedAt, actualResult.GetProperty("LastUpdatedAt").GetDateTime()); + Assert.Equal(expectedResult.RuntimeStatus.ToString(), actualResult.GetProperty("RuntimeStatus").GetString()); + Assert.Equal(expectedResult.SerializedInput, actualResult.GetProperty("SerializedInput").GetString()); + Assert.Equal(expectedResult.SerializedOutput, actualResult.GetProperty("SerializedOutput").GetString()); + Assert.Equal(expectedResult.SerializedCustomStatus, actualResult.GetProperty("SerializedCustomStatus").GetString()); + } + + // Mocks the required HttpRequestData and HttpResponseData for testing purposes. + // This method sets up a mock HttpRequestData with a predefined URL and a mock HttpResponseDatav with a default status code and body. + // The headers of HttpRequestData can be provided as an optional parameter, otherwise an empty HttpHeadersCollection is used. + private HttpRequestData MockHttpRequestAndResponseData(HttpHeadersCollection? headers = null) + { + var mockObjectSerializer = new Mock(); + + // Setup the SerializeAsync method + mockObjectSerializer.Setup(s => s.SerializeAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (stream, value, type, token) => + { + await System.Text.Json.JsonSerializer.SerializeAsync(stream, value, type, cancellationToken: token); + }); + + var workerOptions = new WorkerOptions + { + Serializer = mockObjectSerializer.Object + }; + var mockOptions = new Mock>(); + mockOptions.Setup(o => o.Value).Returns(workerOptions); + + // Mock the service provider + var mockServiceProvider = new Mock(); + + // Set up the service provider to return the mock IOptions + mockServiceProvider.Setup(sp => sp.GetService(typeof(IOptions))) + .Returns(mockOptions.Object); + + // Set up the service provider to return the mock ObjectSerializer + mockServiceProvider.Setup(sp => sp.GetService(typeof(ObjectSerializer))) + .Returns(mockObjectSerializer.Object); + + // Create a mock FunctionContext and assign the service provider + var mockFunctionContext = new Mock(); + mockFunctionContext.SetupGet(c => c.InstanceServices).Returns(mockServiceProvider.Object); + var mockHttpRequestData = new Mock(mockFunctionContext.Object); + + // Set up the URL property. + mockHttpRequestData.SetupGet(r => r.Url).Returns(new Uri("https://localhost:7075/orchestrators/E1_HelloSequence")); + + // If headers are provided, use them, otherwise create a new empty HttpHeadersCollection + headers ??= new HttpHeadersCollection(); + + // Setup the Headers property to return the empty headers + mockHttpRequestData.SetupGet(r => r.Headers).Returns(headers); + + var mockHttpResponseData = new Mock(mockFunctionContext.Object) + { + DefaultValue = DefaultValue.Mock + }; + + // Enable setting StatusCode and Body as mutable properties + mockHttpResponseData.SetupProperty(r => r.StatusCode, HttpStatusCode.OK); + mockHttpResponseData.SetupProperty(r => r.Body, new MemoryStream()); + + // Setup CreateResponse to return the configured HttpResponseData mock + mockHttpRequestData.Setup(r => r.CreateResponse()) + .Returns(mockHttpResponseData.Object); + + return mockHttpRequestData.Object; + } } -} \ No newline at end of file +} diff --git a/test/Worker.Extensions.DurableTask.Tests/Worker.Extensions.DurableTask.Tests.csproj b/test/Worker.Extensions.DurableTask.Tests/Worker.Extensions.DurableTask.Tests.csproj index e49556bb6..e946c176c 100644 --- a/test/Worker.Extensions.DurableTask.Tests/Worker.Extensions.DurableTask.Tests.csproj +++ b/test/Worker.Extensions.DurableTask.Tests/Worker.Extensions.DurableTask.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net6.0 enable enable diff --git a/tools/triageHelper/function_app.py b/tools/triageHelper/function_app.py index 12a6d77ff..c2f4d8aa1 100644 --- a/tools/triageHelper/function_app.py +++ b/tools/triageHelper/function_app.py @@ -12,6 +12,7 @@ "Azure/azure-functions-durable-extension", "Azure/azure-functions-durable-js", "Azure/azure-functions-durable-python", + "Azure/azure-functions-durable-powershell", powershell_worker_repo, "microsoft/durabletask-java", "microsoft/durabletask-dotnet", @@ -40,7 +41,6 @@ def get_triage_issues(repository): 'labels': label, } - payload_str = urllib.parse.urlencode(payload, safe=':+') # Define the GitHub API endpoint api_endpoint = f"https://api.github.com/repos/{repository}/issues" query_str1 = "?labels=Needs%3A%20Triage%20%3Amag%3A"