diff --git a/.github/workflows/codeQL.yml b/.github/workflows/codeQL.yml new file mode 100644 index 000000000..6cb68940e --- /dev/null +++ b/.github/workflows/codeQL.yml @@ -0,0 +1,79 @@ +# This workflow generates weekly CodeQL reports for this repo, a security requirements. +# The workflow is adapted from the following reference: https://github.com/Azure-Samples/azure-functions-python-stream-openai/pull/2/files +# Generic comments on how to modify these file are left intactfor future maintenance. + +name: "CodeQL" + +on: + push: + branches: [ "main", "*" ] # TODO: remove development branch after approval + pull_request: + branches: [ "main", "*"] # TODO: remove development branch after approval + schedule: + - cron: '0 0 * * 1' # Weekly Monday run, needed for weekly reports + workflow_call: # allows to be invoked as part of a larger workflow + workflow_dispatch: # allows for the workflow to run manually see: https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow + +env: + solution: WebJobs.Extensions.DurableTask.sln + config: Release + +jobs: + + analyze: + name: Analyze + runs-on: windows-latest + permissions: + actions: read + contents: read + security-events: write + + + strategy: + fail-fast: false + matrix: + language: ['csharp'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Restore dependencies + run: dotnet restore $solution + + - name: Build + run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true + + # Run CodeQL analysis + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" \ No newline at end of file diff --git a/.github/workflows/smoketest-dotnet-isolated-v4.yml b/.github/workflows/smoketest-dotnet-isolated-v4.yml index f818ff7ae..474f48448 100644 --- a/.github/workflows/smoketest-dotnet-isolated-v4.yml +++ b/.github/workflows/smoketest-dotnet-isolated-v4.yml @@ -19,7 +19,79 @@ jobs: steps: - uses: actions/checkout@v2 - # Validation is blocked on https://github.com/Azure/azure-functions-host/issues/7995 - - name: Run V4 .NET Isolated Smoke Test - run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/DotNetIsolated/Dockerfile -HttpStartPath api/StartHelloCitiesTyped -NoValidation + # Install .NET versions + - name: Set up .NET Core 3.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Set up .NET Core 2.1 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '2.1.x' + + - name: Set up .NET Core 6.x + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.x' + + - name: Set up .NET Core 8.x + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.x' + + # Install Azurite + - name: Set up Node.js (needed for Azurite) + uses: actions/setup-node@v3 + with: + node-version: '18.x' # Azurite requires at least Node 18 + + - name: Install Azurite + run: npm install -g azurite + + - name: Restore WebJobs extension + run: dotnet restore $solution + + - name: Build and pack WebJobs extension + run: cd ./src/WebJobs.Extensions.DurableTask && + mkdir ./out && + dotnet build -c Release WebJobs.Extensions.DurableTask.csproj --output ./out && + mkdir ~/packages && + dotnet nuget push ./out/Microsoft.Azure.WebJobs.Extensions.DurableTask.*.nupkg --source ~/packages && + dotnet nuget add source ~/packages + + - name: Build .NET Isolated Smoke Test + run: cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && + dotnet restore --verbosity normal && + dotnet build -c Release + + - name: Install core tools + run: npm i -g azure-functions-core-tools@4 --unsafe-perm true + + # Run smoke tests + # Unlike other smoke tests, the .NET isolated smoke tests run outside of a docker container, but to race conditions + # when building the smoke test app in docker, causing the build to fail. This is a temporary workaround until the + # root cause is identified and fixed. + + - name: Run smoke tests (Hello Cities) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/StartHelloCitiesTyped + + - name: Run smoke tests (Process Exit) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartProcessExitOrchestrator + + - name: Run smoke tests (Timeout) + shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartTimeoutOrchestrator + + - name: Run smoke tests (OOM) shell: pwsh + run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartOOMOrchestrator \ No newline at end of file diff --git a/Directory.Build.targets b/Directory.Build.targets new file mode 100644 index 000000000..47c2b86a2 --- /dev/null +++ b/Directory.Build.targets @@ -0,0 +1,37 @@ + + + + + + + + + false + <_TranslateUrlPattern>(https://azfunc%40dev\.azure\.com/azfunc/internal/_git|https://dev\.azure\.com/azfunc/internal/_git|https://azfunc\.visualstudio\.com/internal/_git|azfunc%40vs-ssh\.visualstudio\.com:v3/azfunc/internal|git%40ssh\.dev\.azure\.com:v3/azfunc/internal)/([^/\.]+)\.(.+) + <_TranslateUrlReplacement>https://github.com/$2/$3 + + + + + + $([System.Text.RegularExpressions.Regex]::Replace($(ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement))) + + + + $([System.Text.RegularExpressions.Regex]::Replace(%(SourceRoot.ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement))) + + + + + \ No newline at end of file diff --git a/WebJobs.Extensions.DurableTask.sln b/WebJobs.Extensions.DurableTask.sln index 353e83805..b710584c2 100644 --- a/WebJobs.Extensions.DurableTask.sln +++ b/WebJobs.Extensions.DurableTask.sln @@ -18,6 +18,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .editorconfig = .editorconfig azure-pipelines-release-dotnet-isolated.yml = azure-pipelines-release-dotnet-isolated.yml azure-pipelines-release.yml = azure-pipelines-release.yml + Directory.Build.targets = Directory.Build.targets nuget.config = nuget.config README.md = README.md release_notes.md = release_notes.md @@ -94,7 +95,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerfTests", "PerfTests", "{ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DFPerfScenariosV4", "test\DFPerfScenarios\DFPerfScenariosV4.csproj", "{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index d0839ba79..e7a871026 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -6,6 +6,7 @@ trigger: branches: include: - main + - dev # CI only, does not trigger on PRs. pr: none @@ -19,6 +20,7 @@ schedules: branches: include: - main + - dev always: true resources: diff --git a/eng/ci/publish.yml b/eng/ci/publish.yml new file mode 100644 index 000000000..8b65519fa --- /dev/null +++ b/eng/ci/publish.yml @@ -0,0 +1,99 @@ +# This is our package-publishing pipeline. +# When executed, it automatically publishes the output of the 'official pipeline' (the nupkgs) to our internal ADO feed. +# It may optionally also publish the packages to NuGet, but that is gated behind a manual approval. + +trigger: none # only trigger is manual +pr: none # only trigger is manual + +# We include to this variable group to be able to access the NuGet API key +variables: +- group: durabletask_config + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + + pipelines: + - pipeline: officialPipeline # Reference to the pipeline to be used as an artifact source + source: 'durable-extension.official' + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: release + jobs: + + # ADO release + - job: adoRelease + displayName: ADO Release + templateContext: + inputs: + - input: pipelineArtifact + pipeline: officialPipeline # Pipeline reference, as defined in the resources section + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + # The preferred method of release on 1ES is by populating the 'output' section of a 1ES template. + # We use this method to release to ADO, but not to release to NuGet; this is explained in the 'nugetRelease' job. + # To read more about the 'output syntax', see: + # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs + # - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages + outputs: + - output: nuget # 'nuget' is an output "type" for pushing to NuGet + displayName: 'Push to durabletask ADO feed' + packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling + packagesToPush: '$(System.DefaultWorkingDirectory)/**/*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg' + publishVstsFeed: '3f99e810-c336-441f-8892-84983093ad7f/c895696b-ce37-4fe7-b7ce-74333a04f8bf' + allowPackageConflicts: true + + # NuGet approval gate + - job: nugetApproval + displayName: NuGetApproval + pool: server # This task only works when executed on serverl pools, so this needs to be specified + steps: + # Wait for manual approval. + - task: ManualValidation@1 + inputs: + instructions: Confirm you want to push to NuGet + onTimeout: 'reject' + + # NuGet release + - job: nugetRelease + displayName: NuGet Release + dependsOn: + - nugetApproval + - adoRelease + condition: succeeded('nugetApproval', 'adoRelease') + templateContext: + inputs: + - input: pipelineArtifact + pipeline: officialPipeline # Pipeline reference as defined in the resources section + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + # Ideally, we would push to NuGet using the 1ES "template output" syntax, like we do for ADO. + # Unfortunately, that syntax does not allow for skipping duplicates when pushing to NuGet feeds + # (i.e; not failing the job when trying to push a package version that already exists on NuGet). + # This is a problem for us because our pipelines often produce multiple packages, and we want to be able to + # perform a 'nuget push *.nupkg' that skips packages already on NuGet while pushing the rest. + # Therefore, we use a regular .NET Core ADO Task to publish the packages until that usability gap is addressed. + steps: + - task: DotNetCoreCLI@2 + displayName: 'Push to nuget.org' + inputs: + command: custom + custom: nuget + arguments: 'push "*.nupkg" --api-key $(nuget_api_key) --skip-duplicate --source https://api.nuget.org/v3/index.json' + workingDirectory: '$(System.DefaultWorkingDirectory)/drop' \ No newline at end of file diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 7bd8ee166..c68e0d44d 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -46,6 +46,7 @@ jobs: solution: '**/WebJobs.Extensions.DurableTask.sln' vsVersion: "16.0" configuration: Release + msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true # these flags make package build deterministic - template: ci/sign-files.yml@eng parameters: @@ -54,6 +55,13 @@ jobs: pattern: '*DurableTask.dll' signType: dll + - template: ci/sign-files.yml@eng + parameters: + displayName: Sign assemblies + folderPath: 'src/Worker.Extensions.DurableTask/bin/Release' + pattern: '*DurableTask.dll' + signType: dll + # dotnet pack # Packaging needs to be a separate step from build. # This will automatically pick up the signed DLLs. @@ -63,7 +71,20 @@ jobs: command: pack packagesToPack: 'src/**/WebJobs.Extensions.DurableTask.csproj' configuration: Release - packDirectory: 'azure-functions-durable-extension' + packDirectory: $(build.artifactStagingDirectory) + nobuild: true + + + # dotnet pack + # Packaging needs to be a separate step from build. + # This will automatically pick up the signed DLLs. + - task: DotNetCoreCLI@2 + displayName: 'dotnet pack Worker.Extensions.DurableTask.csproj' + inputs: + command: pack + packagesToPack: 'src/**/Worker.Extensions.DurableTask.csproj' + configuration: Release + packDirectory: $(build.artifactStagingDirectory) nobuild: true # Remove redundant symbol package(s) @@ -103,3 +124,14 @@ jobs: SourceFolder: '$(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/Output/' Contents: '**' TargetFolder: '$(System.DefaultWorkingDirectory)/azure-functions-durable-extension/' + + # We also need to build the Java smoke test, for CodeQL compliance + # We don't need to build the other smoke tests, because they can be analyzed without being compiled, + # as they're interpreted languages. + # This could be a separate pipeline, but the task is so small that it's paired with the .NET code build + # for convenience. + - pwsh: | + cd ./test/SmokeTests/OOProcSmokeTests/durableJava/ + gradle build + ls + displayName: 'Build Java OOProc test (for CodeQL compliance)' \ No newline at end of file diff --git a/release_notes.md b/release_notes.md index 11119ac0e..7333dc06a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,19 +1,20 @@ # Release Notes -## Microsoft.Azure.Functions.Worker.Extensions.DurableTask 1.2.1 +## Microsoft.Azure.Functions.Worker.Extensions.DurableTask (version) ### New Features -- Fix regression on `TerminateInstanceAsync` API causing invocations to fail with "unimplemented" exceptions (https://github.com/Azure/azure-functions-durable-extension/pull/2829). - Fail fast if extendedSessionsEnabled set to 'true' for the worker type that doesn't support extended sessions (https://github.com/Azure/azure-functions-durable-extension/pull/2732). ### Bug Fixes +- Fix custom connection name not working when using IDurableClientFactory.CreateClient() - contributed by [@hctan](https://github.com/hctan) + ### Breaking Changes ### Dependency Updates -## Microsoft.Azure.WebJobs.Extensions.DurableTask +## Microsoft.Azure.WebJobs.Extensions.DurableTask 2.13.7 ### New Features @@ -22,3 +23,5 @@ ### Breaking Changes ### Dependency Updates + +- Microsoft.DurableTask.Grpc to 1.3.0 diff --git a/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj index fdd756ee8..d37d305c5 100644 --- a/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj +++ b/samples/durable-client-managed-identity/aspnetcore-app/ToDoList.csproj @@ -6,7 +6,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj b/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj index cb7299f19..34b19725d 100644 --- a/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj +++ b/samples/durable-client-managed-identity/functions-app/DurableClientSampleFunctionApp.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs index d72cd9b48..b3d180153 100644 --- a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs +++ b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.Designer.cs @@ -621,7 +621,7 @@ public static string IOTypesAnalyzerTitle { } /// - /// Looks up a localized string similar to Method call '{0}' violates the orchestrator deterministic code constraint. Methods definied in source code that are used in an orchestrator must be deterministic.. + /// Looks up a localized string similar to Method call '{0}' violates the orchestrator deterministic code constraint. Methods defined in source code that are used in an orchestrator must be deterministic.. /// public static string MethodAnalyzerMessageFormat { get { @@ -630,7 +630,7 @@ public static string MethodAnalyzerMessageFormat { } /// - /// Looks up a localized string similar to Methods definied in source code that are used in an orchestrator must be deterministic.. + /// Looks up a localized string similar to Methods defined in source code that are used in an orchestrator must be deterministic.. /// public static string MethodAnalyzerTitle { get { diff --git a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx index d40722bfb..1eb6cd2a4 100644 --- a/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx +++ b/src/WebJobs.Extensions.DurableTask.Analyzers/Resources.resx @@ -306,10 +306,10 @@ https://docs.microsoft.com/azure/azure-functions/durable/durable-functions-check I/O operations are not allowed inside an orchestrator function. - Method call '{0}' violates the orchestrator deterministic code constraint. Methods definied in source code that are used in an orchestrator must be deterministic. + Method call '{0}' violates the orchestrator deterministic code constraint. Methods defined in source code that are used in an orchestrator must be deterministic. - Methods definied in source code that are used in an orchestrator must be deterministic. + Methods defined in source code that are used in an orchestrator must be deterministic. SignalEntityAsync must use an Entity Interface. diff --git a/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs b/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs index 0162d26b4..51015f79e 100644 --- a/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs +++ b/src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProviderFactory.cs @@ -159,7 +159,7 @@ private AzureStorageDurabilityProvider GetAzureStorageStorageProvider(DurableCli // Need to check this.defaultStorageProvider != null for external clients that call GetDurabilityProvider(attribute) // which never initializes the defaultStorageProvider. if (string.Equals(this.defaultSettings?.TaskHubName, settings.TaskHubName, StringComparison.OrdinalIgnoreCase) && - string.Equals(this.defaultSettings?.StorageConnectionString, settings.StorageConnectionString, StringComparison.OrdinalIgnoreCase) && + string.Equals(this.defaultSettings?.StorageAccountDetails?.ConnectionString, settings.StorageAccountDetails?.ConnectionString, StringComparison.OrdinalIgnoreCase) && this.defaultStorageProvider != null) { // It's important that clients use the same AzureStorageOrchestrationService instance @@ -217,6 +217,7 @@ internal AzureStorageOrchestrationServiceSettings GetAzureStorageOrchestrationSe UseSeparateQueueForEntityWorkItems = this.useSeparateQueueForEntityWorkItems, EntityMessageReorderWindowInMinutes = this.options.EntityMessageReorderWindowInMinutes, MaxEntityOperationBatchSize = this.options.MaxEntityOperationBatchSize, + AllowReplayingTerminalInstances = this.azureStorageOptions.AllowReplayingTerminalInstances, }; if (this.inConsumption) diff --git a/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs b/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs index a0c2fddde..499628929 100644 --- a/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs +++ b/src/WebJobs.Extensions.DurableTask/Bindings/BindingHelper.cs @@ -43,6 +43,7 @@ public string DurableOrchestrationClientToString(IDurableOrchestrationClient cli ConnectionName = attr.ConnectionName, RpcBaseUrl = localRpcAddress, RequiredQueryStringParameters = this.config.HttpApiHandler.GetUniversalQueryStrings(), + HttpBaseUrl = this.config.HttpApiHandler.GetBaseUrl(), }); } @@ -130,6 +131,14 @@ private class OrchestrationClientInputData /// [JsonProperty("rpcBaseUrl")] public string? RpcBaseUrl { get; set; } + + /// + /// The base URL of the Azure Functions host, used in the out-of-proc model. + /// This URL is sent by the client binding object to the Durable Worker extension, + /// allowing the extension to know the host's base URL for constructing management URLs. + /// + [JsonProperty("httpBaseUrl")] + public string? HttpBaseUrl { get; set; } } } } diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs index 9c4d6a02e..86cb21c84 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/RemoteOrchestratorContext.cs @@ -120,6 +120,31 @@ internal void SetResult(string orchestratorResponseJsonText) this.SetResultInternal(result); } + private void ThrowIfPlatformLevelException(FailureDetails failureDetails) + { + // Recursively inspect the FailureDetails of the failed orchestrator and throw if a platform-level exception is detected. + // + // Today, this method only checks for . In the future, we may want to add more cases. + // Other known platform-level exceptions, like timeouts or process exists due to `Environment.FailFast`, do not yield + // a `OrchestratorExecutionResult` as the isolated invocation is abruptly terminated. Therefore, they don't need to be + // handled in this method. + // However, our tests reveal that OOMs are, surprisngly, caught and returned as a `OrchestratorExecutionResult` + // by the isolated process, and thus need special handling. + // + // It's unclear if all OOMs are caught by the isolated process (probably not), and also if there are other platform-level + // errors that are also caught in the isolated process and returned as a `OrchestratorExecutionResult`. Let's add them + // to this method as we encounter them. + if (failureDetails.InnerFailure?.IsCausedBy() ?? false) + { + throw new SessionAbortedException(failureDetails.ErrorMessage); + } + + if (failureDetails.InnerFailure != null) + { + this.ThrowIfPlatformLevelException(failureDetails.InnerFailure); + } + } + private void SetResultInternal(OrchestratorExecutionResult result) { // Look for an orchestration completion action to see if we need to grab the output. @@ -133,6 +158,14 @@ private void SetResultInternal(OrchestratorExecutionResult result) if (completeAction.OrchestrationStatus == OrchestrationStatus.Failed) { + // If the orchestrator failed due to a platform-level error in the isolated process, + // we should re-throw that exception in the host (this process) invocation pipeline, + // so the invocation can be retried. + if (completeAction.FailureDetails != null) + { + this.ThrowIfPlatformLevelException(completeAction.FailureDetails); + } + string message = completeAction switch { { FailureDetails: { } f } => f.ErrorMessage, diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml index 0088042e9..cdf5d8700 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml @@ -93,6 +93,13 @@ HTTP endpoint. For out-of-proc "v2" (middelware passthrough), this is a gRPC endpoint. + + + The base URL of the Azure Functions host, used in the out-of-proc model. + This URL is sent by the client binding object to the Durable Worker extension, + allowing the extension to know the host's base URL for constructing management URLs. + + The result of a clean entity storage operation. @@ -4208,6 +4215,20 @@ A boolean indicating whether to use the table partition strategy. Defaults to false. + + + When false, when an orchestrator is in a terminal state (e.g. Completed, Failed, Terminated), events for that orchestrator are discarded. + Otherwise, events for a terminal orchestrator induce a replay. This may be used to recompute the state of the orchestrator in the "Instances Table". + + + Transactions across Azure Tables are not possible, so we independently update the "History table" and then the "Instances table" + to set the state of the orchestrator. + If a crash were to occur between these two updates, the state of the orchestrator in the "Instances table" would be incorrect. + By setting this configuration to true, you can recover from these inconsistencies by forcing a replay of the orchestrator in response + to a client event like a termination request or an external event, which gives the framework another opportunity to update the state of + the orchestrator in the "Instances table". To force a replay after enabling this configuration, just send any external event to the affected instanceId. + + Throws an exception if the provided hub name violates any naming conventions for the storage provider. diff --git a/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs index 1667aabaf..4a6a506cb 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/AzureStorageOptions.cs @@ -179,6 +179,20 @@ public string TrackingStoreConnectionStringName /// A boolean indicating whether to use the table partition strategy. Defaults to false. public bool UseTablePartitionManagement { get; set; } = false; + /// + /// When false, when an orchestrator is in a terminal state (e.g. Completed, Failed, Terminated), events for that orchestrator are discarded. + /// Otherwise, events for a terminal orchestrator induce a replay. This may be used to recompute the state of the orchestrator in the "Instances Table". + /// + /// + /// Transactions across Azure Tables are not possible, so we independently update the "History table" and then the "Instances table" + /// to set the state of the orchestrator. + /// If a crash were to occur between these two updates, the state of the orchestrator in the "Instances table" would be incorrect. + /// By setting this configuration to true, you can recover from these inconsistencies by forcing a replay of the orchestrator in response + /// to a client event like a termination request or an external event, which gives the framework another opportunity to update the state of + /// the orchestrator in the "Instances table". To force a replay after enabling this configuration, just send any external event to the affected instanceId. + /// + public bool AllowReplayingTerminalInstances { get; set; } = false; + /// /// Throws an exception if the provided hub name violates any naming conventions for the storage provider. /// diff --git a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs index 4d514c670..88a7612dc 100644 --- a/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs +++ b/src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs @@ -138,10 +138,15 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync( byte[] triggerReturnValueBytes = Convert.FromBase64String(triggerReturnValue); P.OrchestratorResponse response = P.OrchestratorResponse.Parser.ParseFrom(triggerReturnValueBytes); + + // TrySetResult may throw if a platform-level error is encountered (like an out of memory exception). context.SetResult( response.Actions.Select(ProtobufUtils.ToOrchestratorAction), response.CustomStatus); + // Here we throw if the orchestrator completed with an application-level error. When we do this, + // the function's result type will be of type `OrchestrationFailureException` which is reserved + // for application-level errors that do not need to be re-tried. context.ThrowIfFailed(); }, #pragma warning restore CS0618 // Type or member is obsolete (not intended for general public use) @@ -159,6 +164,19 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync( // Re-throw so we can abort this invocation. this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested(); } + + // we abort the invocation on "platform level errors" such as: + // - a timeout + // - an out of memory exception + // - a worker process exit + if (functionResult.Exception is Host.FunctionTimeoutException + || functionResult.Exception?.InnerException is SessionAbortedException // see RemoteOrchestrationContext.TrySetResultInternal for details on OOM-handling + || (functionResult.Exception?.InnerException?.GetType().ToString().Contains("WorkerProcessExitException") ?? false)) + { + // TODO: the `WorkerProcessExitException` type is not exposed in our dependencies, it's part of WebJobs.Host.Script. + // Should we add that dependency or should it be exposed in WebJobs.Host? + throw functionResult.Exception; + } } catch (Exception hostRuntimeException) { @@ -238,8 +256,7 @@ await this.LifeCycleNotificationHelper.OrchestratorFailedAsync( else { // the function failed for some other reason - - string exceptionDetails = functionResult.Exception.ToString(); + string exceptionDetails = functionResult.Exception?.ToString() ?? "Framework-internal message: exception details could not be extracted"; this.TraceHelper.FunctionFailed( this.Options.HubName, @@ -258,7 +275,7 @@ await this.LifeCycleNotificationHelper.OrchestratorFailedAsync( orchestratorResult = OrchestratorExecutionResult.ForFailure( message: $"Function '{functionName}' failed with an unhandled exception.", - functionResult.Exception); + functionResult.Exception ?? new Exception($"Function '{functionName}' failed with an unknown unhandled exception")); } // Send the result of the orchestrator function to the DTFx dispatch pipeline. diff --git a/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs b/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs index 57b84012b..d55bce846 100644 --- a/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs +++ b/src/WebJobs.Extensions.DurableTask/ProtobufUtils.cs @@ -97,7 +97,11 @@ public static P.HistoryEvent ToHistoryEventProto(HistoryEvent e) }, }, ScheduledStartTimestamp = startedEvent.ScheduledStartTime == null ? null : Timestamp.FromDateTime(startedEvent.ScheduledStartTime.Value), - CorrelationData = startedEvent.Correlation, + ParentTraceContext = startedEvent.ParentTraceContext == null ? null : new P.TraceContext + { + TraceParent = startedEvent.ParentTraceContext.TraceParent, + TraceState = startedEvent.ParentTraceContext.TraceState, + }, }; break; case EventType.ExecutionTerminated: diff --git a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj index e023c4d4c..05b1a7899 100644 --- a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj +++ b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj @@ -6,7 +6,7 @@ Microsoft.Azure.WebJobs.Extensions.DurableTask 2 13 - 5 + 7 $(PackageSuffix) $(MajorVersion).$(MinorVersion).$(PatchVersion) $(MajorVersion).0.0.0 @@ -107,14 +107,14 @@ - + - + diff --git a/src/Worker.Extensions.DurableTask/AssemblyInfo.cs b/src/Worker.Extensions.DurableTask/AssemblyInfo.cs index 7f387ee55..0be4e12df 100644 --- a/src/Worker.Extensions.DurableTask/AssemblyInfo.cs +++ b/src/Worker.Extensions.DurableTask/AssemblyInfo.cs @@ -5,5 +5,5 @@ using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; // TODO: Find a way to generate this dynamically at build-time -[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.DurableTask", "2.13.4")] -[assembly: InternalsVisibleTo("Worker.Extensions.DurableTask.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] \ No newline at end of file +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.DurableTask", "2.13.7")] +[assembly: InternalsVisibleTo("Worker.Extensions.DurableTask.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] diff --git a/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs b/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs index 1d3da9003..2cfc2706e 100644 --- a/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs +++ b/src/Worker.Extensions.DurableTask/DurableTaskClientConverter.cs @@ -49,7 +49,7 @@ public ValueTask ConvertAsync(ConverterContext context) } DurableTaskClient client = this.clientProvider.GetClient(endpoint, inputData?.taskHubName, inputData?.connectionName); - client = new FunctionsDurableTaskClient(client, inputData!.requiredQueryStringParameters); + client = new FunctionsDurableTaskClient(client, inputData!.requiredQueryStringParameters, inputData!.httpBaseUrl); return new ValueTask(ConversionResult.Success(client)); } catch (Exception innerException) @@ -62,5 +62,5 @@ public ValueTask ConvertAsync(ConverterContext context) } // Serializer is case-sensitive and incoming JSON properties are camel-cased. - private record DurableClientInputData(string rpcBaseUrl, string taskHubName, string connectionName, string requiredQueryStringParameters); + private record DurableClientInputData(string rpcBaseUrl, string taskHubName, string connectionName, string requiredQueryStringParameters, string httpBaseUrl); } diff --git a/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs b/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs index 286c206fe..bbd6222a8 100644 --- a/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs +++ b/src/Worker.Extensions.DurableTask/DurableTaskClientExtensions.cs @@ -120,8 +120,30 @@ public static HttpResponseData CreateCheckStatusResponse( return response; } - private static object SetHeadersAndGetPayload( - DurableTaskClient client, HttpRequestData request, HttpResponseData response, string instanceId) + /// + /// Creates an HTTP management payload for the specified orchestration instance. + /// + /// The . + /// The ID of the orchestration instance. + /// Optional HTTP request data to use for creating the base URL. + /// An object containing instance control URLs. + /// Thrown when instanceId is null or empty. + /// Thrown when a valid base URL cannot be determined. + public static HttpManagementPayload CreateHttpManagementPayload( + this DurableTaskClient client, + string instanceId, + HttpRequestData? request = null) + { + if (string.IsNullOrEmpty(instanceId)) + { + throw new ArgumentException("InstanceId cannot be null or empty.", nameof(instanceId)); + } + + return SetHeadersAndGetPayload(client, request, null, instanceId); + } + + private static HttpManagementPayload SetHeadersAndGetPayload( + DurableTaskClient client, HttpRequestData? request, HttpResponseData? response, string instanceId) { static string BuildUrl(string url, params string?[] queryValues) { @@ -143,22 +165,46 @@ static string BuildUrl(string url, params string?[] queryValues) // request headers into consideration and generate the base URL accordingly. // More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded. // One potential workaround is to set ASPNETCORE_FORWARDEDHEADERS_ENABLED to true. - string baseUrl = request.Url.GetLeftPart(UriPartial.Authority); + + // If HttpRequestData is provided, use its URL; otherwise, get the baseUrl from the DurableTaskClient. + // The base URL could be null if: + // 1. The DurableTaskClient isn't a FunctionsDurableTaskClient (which would have the baseUrl from bindings) + // 2. There's no valid HttpRequestData provided + string? baseUrl = ((request != null) ? request.Url.GetLeftPart(UriPartial.Authority) : GetBaseUrl(client)); + + if (baseUrl == null) + { + throw new InvalidOperationException("Failed to create HTTP management payload as base URL is null. Either use Functions bindings or provide an HTTP request to create the HttpPayload."); + } + + bool isFromRequest = request != null; + string formattedInstanceId = Uri.EscapeDataString(instanceId); - string instanceUrl = $"{baseUrl}/runtime/webhooks/durabletask/instances/{formattedInstanceId}"; + + // The baseUrl differs depending on the source. Eg: + // - From request: http://localhost:7071/ + // - From durable client: http://localhost:7071/runtime/webhooks/durabletask + // We adjust the instanceUrl construction accordingly. + string instanceUrl = isFromRequest + ? $"{baseUrl}/runtime/webhooks/durabletask/instances/{formattedInstanceId}" + : $"{baseUrl}/instances/{formattedInstanceId}"; string? commonQueryParameters = GetQueryParams(client); - response.Headers.Add("Location", BuildUrl(instanceUrl, commonQueryParameters)); - response.Headers.Add("Content-Type", "application/json"); + + if (response != null) + { + response.Headers.Add("Location", BuildUrl(instanceUrl, commonQueryParameters)); + response.Headers.Add("Content-Type", "application/json"); + } - return new + return new HttpManagementPayload { - id = instanceId, - purgeHistoryDeleteUri = BuildUrl(instanceUrl, commonQueryParameters), - sendEventPostUri = BuildUrl($"{instanceUrl}/raiseEvent/{{eventName}}", commonQueryParameters), - statusQueryGetUri = BuildUrl(instanceUrl, commonQueryParameters), - terminatePostUri = BuildUrl($"{instanceUrl}/terminate", "reason={{text}}", commonQueryParameters), - suspendPostUri = BuildUrl($"{instanceUrl}/suspend", "reason={{text}}", commonQueryParameters), - resumePostUri = BuildUrl($"{instanceUrl}/resume", "reason={{text}}", commonQueryParameters) + Id = instanceId, + PurgeHistoryDeleteUri = BuildUrl(instanceUrl, commonQueryParameters), + SendEventPostUri = BuildUrl($"{instanceUrl}/raiseEvent/{{eventName}}", commonQueryParameters), + StatusQueryGetUri = BuildUrl(instanceUrl, commonQueryParameters), + TerminatePostUri = BuildUrl($"{instanceUrl}/terminate", "reason={{text}}", commonQueryParameters), + SuspendPostUri = BuildUrl($"{instanceUrl}/suspend", "reason={{text}}", commonQueryParameters), + ResumePostUri = BuildUrl($"{instanceUrl}/resume", "reason={{text}}", commonQueryParameters) }; } @@ -172,4 +218,9 @@ private static ObjectSerializer GetObjectSerializer(HttpResponseData response) { return client is FunctionsDurableTaskClient functions ? functions.QueryString : null; } + + private static string? GetBaseUrl(DurableTaskClient client) + { + return client is FunctionsDurableTaskClient functions ? functions.HttpBaseUrl : null; + } } diff --git a/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs b/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs index 0f9231375..3c919362d 100644 --- a/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs +++ b/src/Worker.Extensions.DurableTask/FunctionsDurableTaskClient.cs @@ -17,15 +17,16 @@ internal sealed class FunctionsDurableTaskClient : DurableTaskClient { private readonly DurableTaskClient inner; - public FunctionsDurableTaskClient(DurableTaskClient inner, string? queryString) + public FunctionsDurableTaskClient(DurableTaskClient inner, string? queryString, string? httpBaseUrl) : base(inner.Name) { this.inner = inner; this.QueryString = queryString; + this.HttpBaseUrl = httpBaseUrl; } public string? QueryString { get; } - + public string? HttpBaseUrl { get; } public override DurableEntityClient Entities => this.inner.Entities; public override ValueTask DisposeAsync() diff --git a/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs b/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs new file mode 100644 index 000000000..4e7228f87 --- /dev/null +++ b/src/Worker.Extensions.DurableTask/HttpManagementPayload.cs @@ -0,0 +1,97 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// This is a copy of: https://github.com/Azure/azure-functions-durable-extension/blob/dev/src/WebJobs.Extensions.DurableTask/HttpManagementPayload.cs + +using System; +using System.Collections.Generic; +using System.Text; +using Newtonsoft.Json; + +namespace Microsoft.Azure.Functions.Worker; + +/// +/// Data structure containing status, terminate and send external event HTTP endpoints. +/// +public class HttpManagementPayload +{ + /// + /// Gets the ID of the orchestration instance. + /// + /// + /// The ID of the orchestration instance. + /// + [JsonProperty("id")] + public string? Id { get; internal set; } + + /// + /// Gets the HTTP GET status query endpoint URL. + /// + /// + /// The HTTP URL for fetching the instance status. + /// + [JsonProperty("statusQueryGetUri")] + public string? StatusQueryGetUri { get; internal set; } + + /// + /// Gets the HTTP POST external event sending endpoint URL. + /// + /// + /// The HTTP URL for posting external event notifications. + /// + [JsonProperty("sendEventPostUri")] + public string? SendEventPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance termination endpoint. + /// + /// + /// The HTTP URL for posting instance termination commands. + /// + [JsonProperty("terminatePostUri")] + public string? TerminatePostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance rewind endpoint. + /// + /// + /// The HTTP URL for rewinding orchestration instances. + /// + [JsonProperty("rewindPostUri")] + public string? RewindPostUri { get; internal set; } + + /// + /// Gets the HTTP DELETE purge instance history by instance ID endpoint. + /// + /// + /// The HTTP URL for purging instance history by instance ID. + /// + [JsonProperty("purgeHistoryDeleteUri")] + public string? PurgeHistoryDeleteUri { get; internal set; } + + /// + /// Gets the HTTP POST instance restart endpoint. + /// + /// + /// The HTTP URL for restarting an orchestration instance. + /// + [JsonProperty("restartPostUri")] + public string? RestartPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance suspend endpoint. + /// + /// + /// The HTTP URL for suspending an orchestration instance. + /// + [JsonProperty("suspendPostUri")] + public string? SuspendPostUri { get; internal set; } + + /// + /// Gets the HTTP POST instance resume endpoint. + /// + /// + /// The HTTP URL for resuming an orchestration instance. + /// + [JsonProperty("resumePostUri")] + public string? ResumePostUri { get; internal set; } +} diff --git a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj index b5c22a516..0276b570b 100644 --- a/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj +++ b/src/Worker.Extensions.DurableTask/Worker.Extensions.DurableTask.csproj @@ -29,7 +29,7 @@ ..\..\sign.snk - 1.1.4 + 1.1.7 $(VersionPrefix).0 @@ -39,12 +39,13 @@ - - + + + diff --git a/test/Common/CustomTestStorageAccountProvider.cs b/test/Common/CustomTestStorageAccountProvider.cs new file mode 100644 index 000000000..41337555a --- /dev/null +++ b/test/Common/CustomTestStorageAccountProvider.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using DurableTask.AzureStorage; +using Microsoft.WindowsAzure.Storage; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests +{ + internal class CustomTestStorageAccountProvider : IStorageAccountProvider + { + private readonly string customConnectionString; + private readonly string customConnectionName; + + public CustomTestStorageAccountProvider(string connectionName) + { + this.customConnectionName = connectionName; + this.customConnectionString = $"DefaultEndpointsProtocol=https;AccountName=test;AccountKey={GenerateRandomKey()};EndpointSuffix=core.windows.net"; + } + + public CloudStorageAccount GetCloudStorageAccount(string name) => + CloudStorageAccount.Parse(name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString); + + public StorageAccountDetails GetStorageAccountDetails(string name) => + new StorageAccountDetails { ConnectionString = name != this.customConnectionName ? TestHelpers.GetStorageConnectionString() : this.customConnectionString }; + + private static string GenerateRandomKey() + { + string key = Guid.NewGuid().ToString(); + return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(key)); + } + } +} diff --git a/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs b/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs index 30363c325..a55933384 100644 --- a/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs +++ b/test/FunctionsV2/AzureStorageDurabilityProviderFactoryTests.cs @@ -154,5 +154,46 @@ public void EnvironmentIsVMSS_WorkerIdFromEnvironmentVariables() Assert.Equal("waws-prod-euapbn1-003:dw0SmallDedicatedWebWorkerRole_hr0HostRole-3-VM-13", settings.WorkerId); } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void CustomConnectionNameIsResolved() + { + var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection"); + var mockOptions = new OptionsWrapper(new DurableTaskOptions()); + var nameResolver = new Mock().Object; + + var factory = new AzureStorageDurabilityProviderFactory( + mockOptions, + storageAccountProvider, + nameResolver, + NullLoggerFactory.Instance, + TestHelpers.GetMockPlatformInformationService()); + + factory.GetDurabilityProvider(); // This will initialize the default connection string + var provider = factory.GetDurabilityProvider(new DurableClientAttribute() { ConnectionName = "CustomConnection", TaskHub = "TestHubName" }); + + Assert.Equal("CustomConnection", provider.ConnectionName); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void DefaultConnectionNameIsResolved() + { + var storageAccountProvider = new CustomTestStorageAccountProvider("CustomConnection"); + var mockOptions = new OptionsWrapper(new DurableTaskOptions()); + var nameResolver = new Mock().Object; + + var factory = new AzureStorageDurabilityProviderFactory( + mockOptions, + storageAccountProvider, + nameResolver, + NullLoggerFactory.Instance, + TestHelpers.GetMockPlatformInformationService()); + + var provider = factory.GetDurabilityProvider(); + + Assert.Equal("Storage", provider.ConnectionName); + } } } diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln new file mode 100644 index 000000000..a93cc6f6e --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetIsolated", "DotNetIsolated.csproj", "{B2DBA49D-9D25-46DB-8968-15D5E83B4060}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0954D7B4-582F-4F85-AE3E-5D503FB07DB1} + EndGlobalSection +EndGlobal diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs new file mode 100644 index 000000000..8332fa436 --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs @@ -0,0 +1,165 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using System; + +namespace FaultOrchestrators +{ + public static class FaultyOrchestrators + { + [Function(nameof(OOMOrchestrator))] + public static Task OOMOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force an OOM exception on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force the process to run out of memory + List data = new List(); + + for (int i = 0; i < 10000000; i++) + { + data.Add(new byte[1024 * 1024 * 1024]); + } + + // we expect the code to never reach this statement, it should OOM. + // we throw just in case the code does not time out. This should fail the test + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function(nameof(ProcessExitOrchestrator))] + public static Task ProcessExitOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force a sudden process exit on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force sudden crash + Environment.FailFast("Simulating crash!"); + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function(nameof(TimeoutOrchestrator))] + public static Task TimeoutOrchestrator( + [OrchestrationTrigger] TaskOrchestrationContext context) + { + // this orchestrator is not deterministic, on purpose. + // we use the non-determinism to force a timeout on only the first replay + + // check if a file named "replayEvidence" exists in source code directory, create it if it does not. + // From experience, this code runs in `/bin/output/`, so we store the file two directories above. + // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically + // when `func host start` is re-invoked. + string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); + bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); + + if (isTheFirstReplay) + { + System.IO.File.Create(evidenceFile).Close(); + + // force the process to timeout after a 1 minute wait + System.Threading.Thread.Sleep(TimeSpan.FromMinutes(1)); + + // we expect the code to never reach this statement, it should time out. + // we throw just in case the code does not time out. This should fail the test + throw new Exception("this should never be reached"); + } + else { + // if it's not the first replay, delete the evidence file and return + System.IO.File.Delete(evidenceFile); + return Task.CompletedTask; + } + } + + [Function("durable_HttpStartOOMOrchestrator")] + public static async Task HttpStartOOMOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartOOMOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(OOMOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + + [Function("durable_HttpStartProcessExitOrchestrator")] + public static async Task HttpStartProcessExitOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartProcessExitOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(ProcessExitOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + + [Function("durable_HttpStartTimeoutOrchestrator")] + public static async Task HttpStartTimeoutOrchestrator( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext) + { + ILogger logger = executionContext.GetLogger("durable_HttpStartTimeoutOrchestrator"); + + // Function input comes from the request content. + string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + nameof(TimeoutOrchestrator)); + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + } +} diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json index 278b52cde..0ec9c6a89 100644 --- a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/host.json @@ -7,5 +7,14 @@ "excludedTypes": "Request" } } - } + }, + "extensions": { + "durableTask": { + "storageProvider": { + "maxQueuePollingInterval": "00:00:01", + "controlQueueVisibilityTimeout": "00:01:00" + } + } + }, + "functionTimeout": "00:00:30" } \ No newline at end of file diff --git a/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 new file mode 100644 index 000000000..79d679b80 --- /dev/null +++ b/test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 @@ -0,0 +1,119 @@ +# This is a simple test runner to validate the .NET isolated smoke tests. +# It supercedes the usual e2e-tests.ps1 script for the .NET isolated scenario because building the snmoke test app +# on the docker image is unreliable. For more details, see: https://github.com/Azure/azure-functions-host/issues/7995 + +# This script is designed specifically to test cases where the isolated worker process experiences a platform failure: +# timeouts, OOMs, etc. For that reason, it is careful to check that the Functions Host is running and healthy at regular +# intervals. This makes these tests run more slowly than other test categories. + +param( + [Parameter(Mandatory=$true)] + [string]$HttpStartPath +) + +$retryCount = 0; +$statusUrl = $null; +$success = $false; +$haveManuallyRestartedHost = $false; + +Do { + $testIsRunning = $true; + + # Start the functions host if it's not running already. + # Then give it up to 1 minute to start up. + # This is a long wait, but from experience the CI can be slow to start up the host, especially after a platform-error. + $isFunctionsHostRunning = (Get-Process -Name func -ErrorAction SilentlyContinue) + if ($isFunctionsHostRunning -eq $null) { + Write-Host "Starting the Functions host..." -ForegroundColor Yellow + + # The '&' operator is used to run the command in the background + cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 & + Write-Host "Waiting for the Functions host to start up..." -ForegroundColor Yellow + Start-Sleep -Seconds 60 + } + + + try { + # Make sure the Functions runtime is up and running + $pingUrl = "http://localhost:7071/admin/host/ping" + Write-Host "Pinging app at $pingUrl to ensure the host is healthy" -ForegroundColor Yellow + Invoke-RestMethod -Method Post -Uri "http://localhost:7071/admin/host/ping" + Write-Host "Host is healthy!" -ForegroundColor Green + + # Start orchestrator if it hasn't been started yet + if ($statusUrl -eq $null){ + $startOrchestrationUri = "http://localhost:7071/$HttpStartPath" + Write-Host "Starting a new orchestration instance via POST to $startOrchestrationUri..." -ForegroundColor Yellow + + $result = Invoke-RestMethod -Method Post -Uri $startOrchestrationUri + Write-Host "Started orchestration with instance ID '$($result.id)'!" -ForegroundColor Yellow + Write-Host "Waiting for orchestration to complete..." -ForegroundColor Yellow + + $statusUrl = $result.statusQueryGetUri + + # sleep for a bit to give the orchestrator a chance to start, + # then loop once more in case the orchestrator ran quickly, made the host unhealthy, + # and the functions host needs to be restarted + Start-Sleep -Seconds 5 + continue; + } + + # Check the orchestrator status + $result = Invoke-RestMethod -Method Get -Uri $statusUrl + $runtimeStatus = $result.runtimeStatus + Write-Host "Orchestration is $runtimeStatus" -ForegroundColor Yellow + Write-Host $result + + if ($result.runtimeStatus -eq "Completed") { + $success = $true + $testIsRunning = $false + break + } + if ($result.runtimeStatus -eq "Failed") { + $success = $false + $testIsRunning = $false + break + } + + # If the orchestrator did not complete yet, wait for a bit before checking again + Start-Sleep -Seconds 2 + $retryCount = $retryCount + 1 + + } catch { + # we expect to enter this 'catch' block if any of our HTTP requests to the host fail. + # Some failures observed during development include: + # - The host is not running/was restarting/was killed + # - The host is running but not healthy (OOMs may cause this), so it needs to be forcibly restarted + Write-Host "An error occurred:" -ForegroundColor Red + Write-Host $_ -ForegroundColor Red + + # When testing for platform errors, we want to make sure the Functions host is healthy and ready to take requests. + # The Host can get into bad states (for example, in an OOM-inducing test) where it does not self-heal. + # For these cases, we manually restart the host to ensure it is in a good state. We only do this once per test. + if ($haveManuallyRestartedHost -eq $false) { + + # We stop the host process and wait for a bit before checking if it is running again. + Write-Host "Restarting the Functions host..." -ForegroundColor Yellow + Stop-Process -Name "func" -Force + Start-Sleep -Seconds 5 + + # Log whether the process kill succeeded + $haveManuallyRestartedHost = $true + $isFunctionsHostRunning = ((Get-Process -Name func -ErrorAction SilentlyContinue) -eq $null) + Write-Host "Host process killed: $isFunctionsHostRunning" -ForegroundColor Yellow + + # the beginning of the loop will restart the host + continue + } + + # Rethrow the original exception + throw + } + +} while (($testIsRunning -eq $true) -and ($retryCount -lt 65)) + +if ($success -eq $false) { + throw "Orchestration failed or did not compete in time! :(" +} + +Write-Host "Success!" -ForegroundColor Green \ No newline at end of file diff --git a/test/SmokeTests/e2e-test.ps1 b/test/SmokeTests/e2e-test.ps1 index 725988d4c..ab918da8e 100644 --- a/test/SmokeTests/e2e-test.ps1 +++ b/test/SmokeTests/e2e-test.ps1 @@ -26,7 +26,7 @@ function Exit-OnError() { } $ErrorActionPreference = "Stop" -$AzuriteVersion = "3.26.0" +$AzuriteVersion = "3.32.0" if ($NoSetup -eq $false) { # Build the docker image first, since that's the most critical step diff --git a/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs b/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs index 5a335aefa..6f975d2c5 100644 --- a/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs +++ b/test/Worker.Extensions.DurableTask.Tests/FunctionsDurableTaskClientTests.cs @@ -1,5 +1,5 @@ +using Microsoft.Azure.Functions.Worker.Http; using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Client.Grpc; using Moq; namespace Microsoft.Azure.Functions.Worker.Tests @@ -9,7 +9,7 @@ namespace Microsoft.Azure.Functions.Worker.Tests /// public class FunctionsDurableTaskClientTests { - private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient() + private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient(string? baseUrl = null) { // construct mock client @@ -22,7 +22,7 @@ private FunctionsDurableTaskClient GetTestFunctionsDurableTaskClient() It.IsAny(), It.IsAny(), It.IsAny())).Returns(completedTask); DurableTaskClient durableClient = durableClientMock.Object; - FunctionsDurableTaskClient client = new FunctionsDurableTaskClient(durableClient, queryString: null); + FunctionsDurableTaskClient client = new FunctionsDurableTaskClient(durableClient, queryString: null, httpBaseUrl: baseUrl); return client; } @@ -53,5 +53,51 @@ public async void TerminateDoesNotThrow() await client.TerminateInstanceAsync(instanceId, options); await client.TerminateInstanceAsync(instanceId, options, token); } + + /// + /// Test that the `CreateHttpManagementPayload` method returns the expected payload structure without HttpRequestData. + /// + [Fact] + public void CreateHttpManagementPayload_WithBaseUrl() + { + const string BaseUrl = "http://localhost:7071/runtime/webhooks/durabletask"; + FunctionsDurableTaskClient client = this.GetTestFunctionsDurableTaskClient(BaseUrl); + string instanceId = "testInstanceIdWithHostBaseUrl"; + + HttpManagementPayload payload = client.CreateHttpManagementPayload(instanceId); + + AssertHttpManagementPayload(payload, BaseUrl, instanceId); + } + + /// + /// Test that the `CreateHttpManagementPayload` method returns the expected payload structure with HttpRequestData. + /// + [Fact] + public void CreateHttpManagementPayload_WithHttpRequestData() + { + const string requestUrl = "http://localhost:7075/orchestrators/E1_HelloSequence"; + FunctionsDurableTaskClient client = this.GetTestFunctionsDurableTaskClient(); + string instanceId = "testInstanceIdWithRequest"; + + // Create mock HttpRequestData object. + var mockFunctionContext = new Mock(); + var mockHttpRequestData = new Mock(mockFunctionContext.Object); + mockHttpRequestData.SetupGet(r => r.Url).Returns(new Uri(requestUrl)); + + HttpManagementPayload payload = client.CreateHttpManagementPayload(instanceId, mockHttpRequestData.Object); + + AssertHttpManagementPayload(payload, "http://localhost:7075/runtime/webhooks/durabletask", instanceId); + } + + private static void AssertHttpManagementPayload(HttpManagementPayload payload, string BaseUrl, string instanceId) + { + Assert.Equal(instanceId, payload.Id); + Assert.Equal($"{BaseUrl}/instances/{instanceId}", payload.PurgeHistoryDeleteUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/raiseEvent/{{eventName}}", payload.SendEventPostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}", payload.StatusQueryGetUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/terminate?reason={{{{text}}}}", payload.TerminatePostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/suspend?reason={{{{text}}}}", payload.SuspendPostUri); + Assert.Equal($"{BaseUrl}/instances/{instanceId}/resume?reason={{{{text}}}}", payload.ResumePostUri); + } } -} \ No newline at end of file +} diff --git a/tools/triageHelper/function_app.py b/tools/triageHelper/function_app.py index 12a6d77ff..c2f4d8aa1 100644 --- a/tools/triageHelper/function_app.py +++ b/tools/triageHelper/function_app.py @@ -12,6 +12,7 @@ "Azure/azure-functions-durable-extension", "Azure/azure-functions-durable-js", "Azure/azure-functions-durable-python", + "Azure/azure-functions-durable-powershell", powershell_worker_repo, "microsoft/durabletask-java", "microsoft/durabletask-dotnet", @@ -40,7 +41,6 @@ def get_triage_issues(repository): 'labels': label, } - payload_str = urllib.parse.urlencode(payload, safe=':+') # Define the GitHub API endpoint api_endpoint = f"https://api.github.com/repos/{repository}/issues" query_str1 = "?labels=Needs%3A%20Triage%20%3Amag%3A"