Verify idempotency

.github/workflows/integration-tests.yml

@@ -152,11 +152,66 @@ jobs:
         if: always()
         with:
           name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }}
-          path: it/${{ matrix.language }}
+          path: it/${{ matrix.language }}.zip
+
+  idempotency:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          - java
+          - csharp
+          - go
+          - typescript
+          - ruby
+          - php
+          - python
+        description:
+          - "./tests/Kiota.Builder.IntegrationTests/InheritingErrors.yaml"
+          - "./tests/Kiota.Builder.IntegrationTests/NoUnderscoresInModel.yaml"
+          - "oas::petstore"
+          - "apisguru::twitter.com:current"
+          - "apisguru::notion.com"
+          - "apisguru::stripe.com"
+          - "https://raw.githubusercontent.com/googlemaps/openapi-specification/main/dist/google-maps-platform-openapi3.yml"
+          - "apisguru::meraki.com"
+          - "https://developers.pipedrive.com/docs/api/v1/openapi.yaml"
+          - "apisguru::twilio.com:api"
+          - "apisguru::docusign.net"
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: generator
+          path: publish
+      - run: chmod a+x ./publish/kiota
+      - name: Check if test is suppressed
+        id: check-suppressed
+        run: |
+          $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} -kind idempotency
+          Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT
+        shell: pwsh
+      - name: Verify idempotency
+        shell: pwsh
+        run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }}
+        continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }}
+      - id: replace_url
+        if: always()
+        run: |
+          ORIGINAL="${{ matrix.description }}"
+          REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}"
+          echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: idempotency-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }}
+          path: idempotency-folder*.zip
 
   cleanup:
     runs-on: ubuntu-latest
-    needs: [integration]
+    needs: [integration, idempotency]
     steps:
       - uses: jimschubert/delete-artifacts-action@v1
         with:

.gitignore

@@ -448,3 +448,5 @@ reports/
 
 samples/
 it/openapi.yaml
+
+idempotency*.zip

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Multiple fixes to guarantee idempotency of subsequent runs. [#2442](https://github.com/microsoft/kiota/issues/2442)
 - Fix issue with generating classes with Aliases(PHP)[microsoftgraph/msgraph-beta-sdk-php#197](https://github.com/microsoftgraph/msgraph-beta-sdk-php/pull/197)
 - Flattens the models namespaces in Ruby to avoid circular dependencies.
 - Adds ObjectId as a reserved keyword in Ruby to have memory management issues.

it/compare-generation.ps1

@@ -0,0 +1,115 @@
+#!/usr/bin/env pwsh
+
+param(
+    [Parameter(Mandatory = $true)][string]$descriptionUrl,
+    [Parameter(Mandatory = $true)][string]$language,
+    [Parameter(Mandatory = $false)][switch]$dev,
+    [Parameter(Mandatory = $false)][switch]$preserveOutput
+)
+
+if ([string]::IsNullOrEmpty($descriptionUrl)) {
+    Write-Error "Description URL is empty"
+    exit 1
+}
+
+if ([string]::IsNullOrEmpty($language)) {
+    Write-Error "Language is empty"
+    exit 1
+}
+
+function New-TemporaryDirectory {
+    $parent = [System.IO.Path]::GetTempPath()
+    [string] $name = [System.Guid]::NewGuid()
+    New-Item -ItemType Directory -Path (Join-Path $parent $name)
+}
+
+$rootPath = Join-Path -Path $PSScriptRoot -ChildPath ".."
+
+$Env:KIOTA_TUTORIAL_ENABLED = "false"
+$executableName = "kiota"
+if ($IsWindows) {
+    $executableName = "kiota.exe"
+}
+
+switch ($dev) {
+    $true {
+        Write-Warning "Using kiota in dev mode"
+        $kiotaExec = Join-Path -Path $rootPath -ChildPath "src" -AdditionalChildPath "kiota", "bin", "Debug", "net7.0", $executableName
+        break
+    }
+    default { 
+        $kiotaExec = Join-Path -Path $rootPath -ChildPath "publish" -AdditionalChildPath $executableName
+        break
+    }
+}
+
+$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml"
+if (Test-Path $targetOpenapiPath) {
+    Remove-Item $targetOpenapiPath
+}
+
+if ($descriptionUrl.StartsWith("./")) {
+    Copy-Item -Path $descriptionUrl -Destination $targetOpenapiPath -Force
+}
+elseif ($descriptionUrl.StartsWith("http")) {
+    Invoke-WebRequest -Uri $descriptionUrl -OutFile $targetOpenapiPath
+}
+else {
+    Start-Process "$kiotaExec" -ArgumentList "download ${descriptionUrl} --clean-output --output $targetOpenapiPath" -Wait -NoNewWindow    
+}
+
+$tmpFolder1 = New-TemporaryDirectory
+$tmpFolder2 = New-TemporaryDirectory
+
+Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --dvr all --output $tmpFolder1" -Wait -NoNewWindow
+Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --dvr all --output $tmpFolder2" -Wait -NoNewWindow
+
+# Remove variable output files
+Remove-Item (Join-Path -Path $tmpFolder1 -ChildPath "kiota-lock.json")
+if (Test-Path (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log")) {
+    Remove-Item -Force (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log")
+}
+Remove-Item (Join-Path -Path $tmpFolder2 -ChildPath "kiota-lock.json")
+if (Test-Path (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log")) {
+    Remove-Item -Force (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log")
+}
+
+# Compare hashes
+$HashString1 = (Get-ChildItem $tmpFolder1 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
+Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1))
+
+$HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
+Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2))
+
+Write-Output "Folder 1: $tmpFolder1"
+Write-Output "Folder 2: $tmpFolder2"
+
+if ($HashString1 -eq $HashString2) {
+    Write-Output "The content of the folders is identical"
+
+    if (!$preserveOutput) {
+        Remove-Item $tmpFolder1 -Force -Recurse
+        Remove-Item $tmpFolder2 -Force -Recurse
+    }
+
+    Exit 0
+}
+else {
+    Write-Error "The content of the folders is NOT identical"
+    $archivePath1 = Join-Path $rootPath -ChildPath "idempotency-folder1.zip"
+    $archivePath2 = Join-Path $rootPath -ChildPath "idempotency-folder2.zip"
+
+    if (Test-Path $archivePath1) {
+        Remove-Item $archivePath1 -Force -Verbose
+    }
+    if (Test-Path $archivePath2) {
+        Remove-Item $archivePath2 -Force -Verbose
+    }
+
+    if ($dev -eq $false) {
+        Write-Host "Creating archives at location $archivePath1 and $archivePath2"
+        Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1
+        Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2
+    }
+    Exit 1
+}

it/config.json

@@ -154,6 +154,12 @@
         "Language": "python",
         "Rationale": "https://github.com/microsoft/kiota/issues/2957"
       }
+    ],
+    "IdempotencySuppressions": [
+      {
+        "Language": "typescript",
+        "Rationale": "https://github.com/microsoft/kiota/issues/1812"
+      }
     ]
   },
   "apisguru::twilio.com:api": {
@@ -216,6 +222,12 @@
         "Language": "ruby",
         "Rationale": "https://github.com/microsoft/kiota/issues/2484"
       }
+    ],
+    "IdempotencySuppressions": [
+      {
+        "Language": "ruby",
+        "Rationale": "https://github.com/microsoft/kiota/issues/1816"
+      }
     ]
   },
   "apisguru::stripe.com": {
@@ -244,6 +256,36 @@
         "Language": "ruby",
         "Rationale": "https://github.com/microsoft/kiota/issues/1816"
       }
+    ],
+    "IdempotencySuppressions": [
+      {
+        "Language": "typescript",
+        "Rationale": "https://github.com/microsoft/kiota/issues/1812"
+      },
+      {
+        "Language": "go",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      },
+      {
+        "Language": "php",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      },
+      {
+        "Language": "java",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      },
+      {
+        "Language": "csharp",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      },
+      {
+        "Language": "ruby",
+        "Rationale": "https://github.com/microsoft/kiota/issues/1816"
+      },
+      {
+        "Language": "python",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      }
     ]
   },
   "apisguru::meraki.com": {
@@ -280,6 +322,16 @@
         "Language": "ruby",
         "Rationale": "https://github.com/microsoft/kiota/issues/1816"
       }
+    ],
+    "IdempotencySuppressions": [
+      {
+        "Language": "php",
+        "Rationale": "https://github.com/microsoft/kiota/issues/3067"
+      },
+      {
+        "Language": "typescript",
+        "Rationale": "https://github.com/microsoft/kiota/issues/1812"
+      }
     ]
   }
 }

it/do-clean.ps1

@@ -9,8 +9,7 @@ if ([string]::IsNullOrEmpty($language)) {
     exit 1
 }
 
-$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
-$testPath = Join-Path -Path $scriptPath -ChildPath $language
+$testPath = Join-Path -Path $PSScriptRoot -ChildPath $language
 
 Push-Location $testPath
 if ($language -eq "csharp") {
@@ -30,3 +29,9 @@ elseif ($language -eq "php") {
     Remove-Item composer.lock -ErrorAction SilentlyContinue
 }
 Pop-Location
+
+$archiveLocation = Join-Path -Path $PSScriptRoot -ChildPath "$language.zip"
+if (Test-Path $archiveLocation) {
+    Remove-Item $archiveLocation -Force -ErrorAction SilentlyContinue -Verbose
+}
+Compress-Archive -Path $testPath -DestinationPath $archiveLocation

it/exec-cmd.ps1

@@ -28,38 +28,33 @@ function Invoke-Call {
     }
 }
 
-function Retry([Action]$action)
-{
-    $attempts=10
-    $sleepInSeconds=1
-    do
-    {
-        try
-        {
+function Retry([Action]$action) {
+    $attempts = 10
+    $sleepInSeconds = 1
+    do {
+        try {
             $action.Invoke();
             break;
         }
-        catch [Exception]
-        {
+        catch [Exception] {
             Write-Host $_.Exception.Message
         }            
         $attempts--
         if ($attempts -gt 0) { sleep $sleepInSeconds }
     } while ($attempts -gt 0)
 }
 
-$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
-$testPath = Join-Path -Path $scriptPath -ChildPath $language
-$mockServerPath = Join-Path -Path $scriptPath -ChildPath "mockserver"
+$testPath = Join-Path -Path $PSScriptRoot -ChildPath $language
+$mockServerPath = Join-Path -Path $PSScriptRoot -ChildPath "mockserver"
 
 function Kill-MockServer {
     Push-Location $mockServerPath
-        mvn --batch-mode mockserver:stopForked
+    mvn --batch-mode mockserver:stopForked
     Pop-Location
 }
 
 $mockSeverITFolder = $null
-$configPath = Join-Path -Path $scriptPath -ChildPath "config.json"
+$configPath = Join-Path -Path $PSScriptRoot -ChildPath "config.json"
 $jsonValue = Get-Content -Path $configPath -Raw | ConvertFrom-Json
 $descriptionValue = $jsonValue.psobject.properties.Where({ $_.name -eq $descriptionUrl }).value
 if ($null -ne $descriptionValue) {
@@ -73,14 +68,14 @@ if (!([string]::IsNullOrEmpty($mockSeverITFolder))) {
     # Kill any leftover MockServer
     Kill-MockServer
     Push-Location $mockServerPath
-        mvn  --batch-mode mockserver:runForked
+    mvn  --batch-mode mockserver:runForked
     Pop-Location
 
     # Provision Mock server with the right spec
-    $openapiUrl = Join-Path -Path $scriptPath -ChildPath "openapi.yaml"
+    $openapiUrl = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml"
 
     # provision MockServer to mock the specific openapi description https://www.mock-server.com/mock_server/using_openapi.html#button_open_api_filepath
-    Retry({Invoke-WebRequest -Method PUT -Body "{ `"specUrlOrPayload`": `"$openapiUrl`" }" -Uri http://localhost:1080/mockserver/openapi -ContentType application/json})
+    Retry({ Invoke-WebRequest -Method PUT -Body "{ `"specUrlOrPayload`": `"$openapiUrl`" }" -Uri http://localhost:1080/mockserver/openapi -ContentType application/json })
 }
 
 Push-Location $testPath
@@ -101,7 +96,8 @@ if ($language -eq "csharp") {
         } -ErrorAction Stop
 
         Pop-Location
-    } else {
+    }
+    else {
         Invoke-Call -ScriptBlock {
             dotnet build
         } -ErrorAction Stop
@@ -124,7 +120,8 @@ elseif ($language -eq "java") {
         } -ErrorAction Stop
 
         Pop-Location
-    } else {
+    }
+    else {
         Invoke-Call -ScriptBlock {
             mvn clean compile --batch-mode
         } -ErrorAction Stop
@@ -147,7 +144,8 @@ elseif ($language -eq "go") {
         } -ErrorAction Stop
 
         Pop-Location
-    } else {
+    }
+    else {
         Invoke-Call -ScriptBlock {
             go install
             go build