Skip to content

Commit

Permalink
Merge branch 'dev' into fail-fast-if-incorrect-extended-sessions
Browse files Browse the repository at this point in the history
  • Loading branch information
cgillum authored Oct 28, 2024
2 parents 87b283a + a7c6d69 commit 7ec3194
Show file tree
Hide file tree
Showing 35 changed files with 1,064 additions and 52 deletions.
79 changes: 79 additions & 0 deletions .github/workflows/codeQL.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# This workflow generates weekly CodeQL reports for this repo, a security requirements.
# The workflow is adapted from the following reference: https://github.com/Azure-Samples/azure-functions-python-stream-openai/pull/2/files
# Generic comments on how to modify these file are left intactfor future maintenance.

name: "CodeQL"

on:
push:
branches: [ "main", "*" ] # TODO: remove development branch after approval
pull_request:
branches: [ "main", "*"] # TODO: remove development branch after approval
schedule:
- cron: '0 0 * * 1' # Weekly Monday run, needed for weekly reports
workflow_call: # allows to be invoked as part of a larger workflow
workflow_dispatch: # allows for the workflow to run manually see: https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow

env:
solution: WebJobs.Extensions.DurableTask.sln
config: Release

jobs:

analyze:
name: Analyze
runs-on: windows-latest
permissions:
actions: read
contents: read
security-events: write


strategy:
fail-fast: false
matrix:
language: ['csharp']
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

steps:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.

# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality

- uses: actions/checkout@v3
with:
submodules: true

- name: Setup .NET
uses: actions/setup-dotnet@v3

- name: Set up .NET Core 2.1
uses: actions/setup-dotnet@v3
with:
dotnet-version: '2.1.x'

- name: Set up .NET Core 3.1
uses: actions/setup-dotnet@v3
with:
dotnet-version: '3.1.x'

- name: Restore dependencies
run: dotnet restore $solution

- name: Build
run: dotnet build $solution #--configuration $config #--no-restore -p:FileVersionRevision=$GITHUB_RUN_NUMBER -p:ContinuousIntegrationBuild=true

# Run CodeQL analysis
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
78 changes: 75 additions & 3 deletions .github/workflows/smoketest-dotnet-isolated-v4.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,79 @@ jobs:
steps:
- uses: actions/checkout@v2

# Validation is blocked on https://github.com/Azure/azure-functions-host/issues/7995
- name: Run V4 .NET Isolated Smoke Test
run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/DotNetIsolated/Dockerfile -HttpStartPath api/StartHelloCitiesTyped -NoValidation
# Install .NET versions
- name: Set up .NET Core 3.1
uses: actions/setup-dotnet@v3
with:
dotnet-version: '3.1.x'

- name: Set up .NET Core 2.1
uses: actions/setup-dotnet@v3
with:
dotnet-version: '2.1.x'

- name: Set up .NET Core 6.x
uses: actions/setup-dotnet@v3
with:
dotnet-version: '6.x'

- name: Set up .NET Core 8.x
uses: actions/setup-dotnet@v3
with:
dotnet-version: '8.x'

# Install Azurite
- name: Set up Node.js (needed for Azurite)
uses: actions/setup-node@v3
with:
node-version: '18.x' # Azurite requires at least Node 18

- name: Install Azurite
run: npm install -g azurite

- name: Restore WebJobs extension
run: dotnet restore $solution

- name: Build and pack WebJobs extension
run: cd ./src/WebJobs.Extensions.DurableTask &&
mkdir ./out &&
dotnet build -c Release WebJobs.Extensions.DurableTask.csproj --output ./out &&
mkdir ~/packages &&
dotnet nuget push ./out/Microsoft.Azure.WebJobs.Extensions.DurableTask.*.nupkg --source ~/packages &&
dotnet nuget add source ~/packages

- name: Build .NET Isolated Smoke Test
run: cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated &&
dotnet restore --verbosity normal &&
dotnet build -c Release

- name: Install core tools
run: npm i -g azure-functions-core-tools@4 --unsafe-perm true

# Run smoke tests
# Unlike other smoke tests, the .NET isolated smoke tests run outside of a docker container, but to race conditions
# when building the smoke test app in docker, causing the build to fail. This is a temporary workaround until the
# root cause is identified and fixed.

- name: Run smoke tests (Hello Cities)
shell: pwsh
run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 &
./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/StartHelloCitiesTyped

- name: Run smoke tests (Process Exit)
shell: pwsh
run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartProcessExitOrchestrator

- name: Run smoke tests (Timeout)
shell: pwsh
run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 &
./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartTimeoutOrchestrator

- name: Run smoke tests (OOM)
shell: pwsh
run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
cd ./test/SmokeTests/OOProcSmokeTests/DotNetIsolated && func host start --port 7071 &
./test/SmokeTests/OOProcSmokeTests/DotNetIsolated/run-smoke-tests.ps1 -HttpStartPath api/durable_HttpStartOOMOrchestrator
37 changes: 37 additions & 0 deletions Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<Project>
<!-- This is copied from:https://github.com/Azure/azure-functions-host/blob/dev/eng/build/RepositoryInfo.targets -->
<!-- The following build target allows us to reconstruct source-link information when building in 1ES -->

<!--
The convention for names of Azure DevOps repositories mirrored from GitHub is "{GitHub org name}.{GitHub repository name}".
-->
<PropertyGroup>
<!-- There are quite a few git repo forms:
https://[email protected]/azfunc/internal/_git/azure.azure-functions-host
https://dev.azure.com/azfunc/internal/_git/azure.azure-functions-host
https://azfunc.visualstudio.com/internal/_git/azure.azure-functions-host
[email protected]:v3/azfunc/internal/azure.azure-functions-host
[email protected]:v3/azfunc/internal/azure.azure-functions-host
-->
<!-- Set DisableSourceLinkUrlTranslation to true when building a tool for internal use where sources only come from internal URIs -->
<DisableSourceLinkUrlTranslation Condition="'$(DisableSourceLinkUrlTranslation)' == ''">false</DisableSourceLinkUrlTranslation>
<_TranslateUrlPattern>(https://azfunc%40dev\.azure\.com/azfunc/internal/_git|https://dev\.azure\.com/azfunc/internal/_git|https://azfunc\.visualstudio\.com/internal/_git|azfunc%40vs-ssh\.visualstudio\.com:v3/azfunc/internal|git%40ssh\.dev\.azure\.com:v3/azfunc/internal)/([^/\.]+)\.(.+)</_TranslateUrlPattern>
<_TranslateUrlReplacement>https://github.com/$2/$3</_TranslateUrlReplacement>
</PropertyGroup>

<!-- When building from Azure Devops we update SourceLink to point back to the GitHub repo. -->
<Target Name="_TranslateAzureDevOpsUrlToGitHubUrl"
Condition="'$(DisableSourceLinkUrlTranslation)' == 'false'"
DependsOnTargets="$(SourceControlManagerUrlTranslationTargets)"
BeforeTargets="SourceControlManagerPublishTranslatedUrls">
<PropertyGroup>
<ScmRepositoryUrl>$([System.Text.RegularExpressions.Regex]::Replace($(ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement)))</ScmRepositoryUrl>
</PropertyGroup>
<ItemGroup>
<SourceRoot Update="@(SourceRoot)">
<ScmRepositoryUrl>$([System.Text.RegularExpressions.Regex]::Replace(%(SourceRoot.ScmRepositoryUrl), $(_TranslateUrlPattern), $(_TranslateUrlReplacement)))</ScmRepositoryUrl>
</SourceRoot>
</ItemGroup>
</Target>

</Project>
3 changes: 2 additions & 1 deletion WebJobs.Extensions.DurableTask.sln
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
.editorconfig = .editorconfig
azure-pipelines-release-dotnet-isolated.yml = azure-pipelines-release-dotnet-isolated.yml
azure-pipelines-release.yml = azure-pipelines-release.yml
Directory.Build.targets = Directory.Build.targets
nuget.config = nuget.config
README.md = README.md
release_notes.md = release_notes.md
Expand Down Expand Up @@ -94,7 +95,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerfTests", "PerfTests", "{
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DFPerfScenariosV4", "test\DFPerfScenarios\DFPerfScenariosV4.csproj", "{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.DurableTask.Tests", "test\Worker.Extensions.DurableTask.Tests\Worker.Extensions.DurableTask.Tests.csproj", "{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand Down
2 changes: 2 additions & 0 deletions eng/ci/official-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ trigger:
branches:
include:
- main
- dev

# CI only, does not trigger on PRs.
pr: none
Expand All @@ -19,6 +20,7 @@ schedules:
branches:
include:
- main
- dev
always: true

resources:
Expand Down
99 changes: 99 additions & 0 deletions eng/ci/publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# This is our package-publishing pipeline.
# When executed, it automatically publishes the output of the 'official pipeline' (the nupkgs) to our internal ADO feed.
# It may optionally also publish the packages to NuGet, but that is gated behind a manual approval.

trigger: none # only trigger is manual
pr: none # only trigger is manual

# We include to this variable group to be able to access the NuGet API key
variables:
- group: durabletask_config

resources:
repositories:
- repository: 1es
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
- repository: eng
type: git
name: engineering
ref: refs/tags/release

pipelines:
- pipeline: officialPipeline # Reference to the pipeline to be used as an artifact source
source: 'durable-extension.official'

extends:
template: v1/1ES.Official.PipelineTemplate.yml@1es
parameters:
pool:
name: 1es-pool-azfunc
image: 1es-windows-2022
os: windows

stages:
- stage: release
jobs:

# ADO release
- job: adoRelease
displayName: ADO Release
templateContext:
inputs:
- input: pipelineArtifact
pipeline: officialPipeline # Pipeline reference, as defined in the resources section
artifactName: drop
targetPath: $(System.DefaultWorkingDirectory)/drop

# The preferred method of release on 1ES is by populating the 'output' section of a 1ES template.
# We use this method to release to ADO, but not to release to NuGet; this is explained in the 'nugetRelease' job.
# To read more about the 'output syntax', see:
# - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs
# - https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages
outputs:
- output: nuget # 'nuget' is an output "type" for pushing to NuGet
displayName: 'Push to durabletask ADO feed'
packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling
packagesToPush: '$(System.DefaultWorkingDirectory)/**/*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg'
publishVstsFeed: '3f99e810-c336-441f-8892-84983093ad7f/c895696b-ce37-4fe7-b7ce-74333a04f8bf'
allowPackageConflicts: true

# NuGet approval gate
- job: nugetApproval
displayName: NuGetApproval
pool: server # This task only works when executed on serverl pools, so this needs to be specified
steps:
# Wait for manual approval.
- task: ManualValidation@1
inputs:
instructions: Confirm you want to push to NuGet
onTimeout: 'reject'

# NuGet release
- job: nugetRelease
displayName: NuGet Release
dependsOn:
- nugetApproval
- adoRelease
condition: succeeded('nugetApproval', 'adoRelease')
templateContext:
inputs:
- input: pipelineArtifact
pipeline: officialPipeline # Pipeline reference as defined in the resources section
artifactName: drop
targetPath: $(System.DefaultWorkingDirectory)/drop
# Ideally, we would push to NuGet using the 1ES "template output" syntax, like we do for ADO.
# Unfortunately, that syntax does not allow for skipping duplicates when pushing to NuGet feeds
# (i.e; not failing the job when trying to push a package version that already exists on NuGet).
# This is a problem for us because our pipelines often produce multiple packages, and we want to be able to
# perform a 'nuget push *.nupkg' that skips packages already on NuGet while pushing the rest.
# Therefore, we use a regular .NET Core ADO Task to publish the packages until that usability gap is addressed.
steps:
- task: DotNetCoreCLI@2
displayName: 'Push to nuget.org'
inputs:
command: custom
custom: nuget
arguments: 'push "*.nupkg" --api-key $(nuget_api_key) --skip-duplicate --source https://api.nuget.org/v3/index.json'
workingDirectory: '$(System.DefaultWorkingDirectory)/drop'
34 changes: 33 additions & 1 deletion eng/templates/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ jobs:
solution: '**/WebJobs.Extensions.DurableTask.sln'
vsVersion: "16.0"
configuration: Release
msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true # these flags make package build deterministic

- template: ci/sign-files.yml@eng
parameters:
Expand All @@ -54,6 +55,13 @@ jobs:
pattern: '*DurableTask.dll'
signType: dll

- template: ci/sign-files.yml@eng
parameters:
displayName: Sign assemblies
folderPath: 'src/Worker.Extensions.DurableTask/bin/Release'
pattern: '*DurableTask.dll'
signType: dll

# dotnet pack
# Packaging needs to be a separate step from build.
# This will automatically pick up the signed DLLs.
Expand All @@ -63,7 +71,20 @@ jobs:
command: pack
packagesToPack: 'src/**/WebJobs.Extensions.DurableTask.csproj'
configuration: Release
packDirectory: 'azure-functions-durable-extension'
packDirectory: $(build.artifactStagingDirectory)
nobuild: true


# dotnet pack
# Packaging needs to be a separate step from build.
# This will automatically pick up the signed DLLs.
- task: DotNetCoreCLI@2
displayName: 'dotnet pack Worker.Extensions.DurableTask.csproj'
inputs:
command: pack
packagesToPack: 'src/**/Worker.Extensions.DurableTask.csproj'
configuration: Release
packDirectory: $(build.artifactStagingDirectory)
nobuild: true

# Remove redundant symbol package(s)
Expand Down Expand Up @@ -103,3 +124,14 @@ jobs:
SourceFolder: '$(System.DefaultWorkingDirectory)/test/PerfTests/DFPerfTests/Output/'
Contents: '**'
TargetFolder: '$(System.DefaultWorkingDirectory)/azure-functions-durable-extension/'

# We also need to build the Java smoke test, for CodeQL compliance
# We don't need to build the other smoke tests, because they can be analyzed without being compiled,
# as they're interpreted languages.
# This could be a separate pipeline, but the task is so small that it's paired with the .NET code build
# for convenience.
- pwsh: |
cd ./test/SmokeTests/OOProcSmokeTests/durableJava/
gradle build
ls
displayName: 'Build Java OOProc test (for CodeQL compliance)'
Loading

0 comments on commit 7ec3194

Please sign in to comment.