Add GitHub Action for E2E tests (#3017)

* Add GitHub Action for E2E tests
* Many bugfixes in build script for compatibility with GH action workers

.github/workflows/E2ETest.yml

@@ -0,0 +1,52 @@
+name: End to End Test - .NET Isolated/Func V4
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main, dev, andystaples/add-gh-action-for-e2e-tests ]
+    paths:
+      - 'src/**'
+      - 'test/e2e/**'
+  pull_request:
+    branches: [ main, dev, andystaples/add-gh-action-for-e2e-tests ]
+    paths:
+      - 'src/**'
+      - 'test/e2e/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 6.0.x
+
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 8.0.x
+
+      - name: Setup E2E tests
+        shell: pwsh
+        run: |
+          .\test\e2e\Tests\build-e2e-test.ps1 -SkipStorageEmulator
+
+      - name: Build
+        working-directory: test/e2e/Tests
+        run: dotnet build
+
+      # Install Azurite
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x' # Azurite requires at least Node 18
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Run E2E tests
+        working-directory: test/e2e/Tests
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 & dotnet test

test/e2e/Apps/BasicDotNetIsolated/local.settings.json

@@ -0,0 +1,7 @@
+{
+    "IsEncrypted": false,
+    "Values": {
+        "AzureWebJobsStorage": "UseDevelopmentStorage=true",
+        "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated"
+    }
+}

test/e2e/Tests/Tests/HelloCitiesTest.cs

@@ -20,6 +20,20 @@ public HttpEndToEndTests(FunctionAppFixture fixture, ITestOutputHelper testOutpu
         _output = testOutputHelper;
     }
 
+    // Due to some kind of asynchronous race condition in XUnit, when running these tests in pipelines,
+    // the output may be disposed before the message is written. Just ignore these types of errors for now. 
+    private void WriteOutput(string message)
+    {
+        try
+        {
+            _output.WriteLine(message);
+        }
+        catch
+        {
+            // Ignore
+        }
+    }
+
     [Theory]
     [InlineData("HelloCities_HttpStart", HttpStatusCode.Accepted, "Hello Tokyo!")]
     public async Task HttpTriggerTests(string functionName, HttpStatusCode expectedStatusCode, string partialExpectedOutput)
@@ -36,7 +50,7 @@ public async Task HttpTriggerTests(string functionName, HttpStatusCode expectedS
     }
 
     [Theory]
-    [InlineData("HelloCities_HttpStart_Scheduled", 5, HttpStatusCode.Accepted)]
+    [InlineData("HelloCities_HttpStart_Scheduled", 10, HttpStatusCode.Accepted)]
     [InlineData("HelloCities_HttpStart_Scheduled", -5, HttpStatusCode.Accepted)]
     public async Task ScheduledStartTests(string functionName, int startDelaySeconds, HttpStatusCode expectedStatusCode)
     {
@@ -52,19 +66,25 @@ public async Task ScheduledStartTests(string functionName, int startDelaySeconds
         Assert.Equal(expectedStatusCode, response.StatusCode);
 
         var orchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
-        while (DateTime.UtcNow < scheduledStartTime)
+        while (DateTime.UtcNow < scheduledStartTime + TimeSpan.FromSeconds(-1))
         {
-            _output.WriteLine($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}");
+            WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}");
             orchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
             Assert.Equal("Pending", orchestrationDetails.RuntimeStatus);
-            Thread.Sleep(3000);
+            Thread.Sleep(1000);
         }
 
         // Give a small amount of time for the orchestration to complete, even if scheduled to run immediately
-        Thread.Sleep(1000);
-        _output.WriteLine($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}, looking for completed");
-
+        Thread.Sleep(3000);
+        WriteOutput($"Test scheduled for {scheduledStartTime}, current time {DateTime.Now}, looking for completed");
         var finalOrchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+        int retryAttempts = 0;
+        while (finalOrchestrationDetails.RuntimeStatus != "Completed" && retryAttempts < 10)
+        {
+            Thread.Sleep(1000);
+            finalOrchestrationDetails = DurableHelpers.GetRunningOrchestrationDetails(statusQueryGetUri);
+            retryAttempts++;
+        }
         Assert.Equal("Completed", finalOrchestrationDetails.RuntimeStatus);
 
         Assert.True(finalOrchestrationDetails.LastUpdatedTime > scheduledStartTime);