From c56e6e872d61e00e1e5f594e47dc47b704678576 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 11:53:19 +0200 Subject: [PATCH 01/30] minor --- .github/workflows/integration-tests.yml | 148 ++++++++++++------------ it/compare-generation.ps1 | 89 ++++++++++++++ 2 files changed, 166 insertions(+), 71 deletions(-) create mode 100755 it/compare-generation.ps1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 69f78ba9b6..defb6fe897 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,30 +1,31 @@ name: integration-tests on: - workflow_dispatch: push: - branches: [main] - paths-ignore: - [ - "abstractions/**", - "authentication/**", - "serialization/**", - "http/**", - "**.md", - ".vscode/**", - "**.svg", - ] - pull_request: - paths-ignore: - [ - "abstractions/**", - "authentication/**", - "serialization/**", - "http/**", - "**.md", - ".vscode/**", - "**.svg", - ] + # workflow_dispatch: + # push: + # branches: [main] + # paths-ignore: + # [ + # "abstractions/**", + # "authentication/**", + # "serialization/**", + # "http/**", + # "**.md", + # ".vscode/**", + # "**.svg", + # ] + # pull_request: + # paths-ignore: + # [ + # "abstractions/**", + # "authentication/**", + # "serialization/**", + # "http/**", + # "**.md", + # ".vscode/**", + # "**.svg", + # ] concurrency: # Only run once for latest commit per ref and cancel other (previous) runs. @@ -90,56 +91,61 @@ jobs: distribution: "temurin" cache: "maven" - - name: Setup .NET - if: matrix.language == 'csharp' - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 7.0.x - - name: Setup Go - if: matrix.language == 'go' - uses: actions/setup-go@v4 - with: - go-version: "^1.20" - - name: Setup Typescript - if: matrix.language == 'typescript' - uses: actions/setup-node@v3 - with: - node-version: "18" - - name: Setup Ruby - if: matrix.language == 'ruby' - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2.0" - bundler-cache: true - - name: Setup PHP - if: matrix.language == 'php' - uses: shivammathur/setup-php@v2 - with: - php-version: "8.2" - coverage: xdebug - - name: Setup Python - if: matrix.language == 'python' - uses: actions/setup-python@v4 - with: - python-version: "3.11" + # - name: Setup .NET + # if: matrix.language == 'csharp' + # uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: 7.0.x + # - name: Setup Go + # if: matrix.language == 'go' + # uses: actions/setup-go@v4 + # with: + # go-version: "^1.20" + # - name: Setup Typescript + # if: matrix.language == 'typescript' + # uses: actions/setup-node@v3 + # with: + # node-version: "18" + # - name: Setup Ruby + # if: matrix.language == 'ruby' + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.2.0" + # bundler-cache: true + # - name: Setup PHP + # if: matrix.language == 'php' + # uses: shivammathur/setup-php@v2 + # with: + # php-version: "8.2" + # coverage: xdebug + # - name: Setup Python + # if: matrix.language == 'python' + # uses: actions/setup-python@v4 + # with: + # python-version: "3.11" - - name: Check if test is suppressed - id: check-suppressed - run: | - $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT - shell: pwsh - - name: Generate Code - shell: pwsh - run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - - name: Execute the IT test - shell: pwsh - run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - - name: Cleanup the IT folder + # - name: Check if test is suppressed + # id: check-suppressed + # run: | + # $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT + # shell: pwsh + # - name: Generate Code + # shell: pwsh + # run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + # - name: Execute the IT test + # shell: pwsh + # run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + # - name: Cleanup the IT folder + # shell: pwsh + # run: ./it/do-clean.ps1 -language ${{ matrix.language }} + # if: always() + + - name: Verify idempotency shell: pwsh - run: ./it/do-clean.ps1 -language ${{ matrix.language }} + run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} if: always() - id: replace_url diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 new file mode 100755 index 0000000000..3201cf8a01 --- /dev/null +++ b/it/compare-generation.ps1 @@ -0,0 +1,89 @@ +#!/usr/bin/env pwsh + +param( + [Parameter(Mandatory = $true)][string]$descriptionUrl, + [Parameter(Mandatory = $true)][string]$language, + [Parameter(Mandatory = $false)][switch]$dev +) + +if ([string]::IsNullOrEmpty($descriptionUrl)) { + Write-Error "Description URL is empty" + exit 1 +} + +if ([string]::IsNullOrEmpty($language)) { + Write-Error "Language is empty" + exit 1 +} + +function New-TemporaryDirectory { + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + New-Item -ItemType Directory -Path (Join-Path $parent $name) +} + +$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$rootPath = Join-Path -Path $scriptPath -ChildPath ".." + +$executableName = "kiota" +if ($IsWindows) { + $executableName = "kiota.exe" +} + +switch ($dev) { + $true { + Write-Warning "Using kiota in dev mode" + $kiotaExec = Join-Path -Path $rootPath -ChildPath "src" -AdditionalChildPath "kiota", "bin", "Debug", "net7.0", $executableName + break + } + default { + $kiotaExec = Join-Path -Path $rootPath -ChildPath "publish" -AdditionalChildPath $executableName + break + } +} + +$targetOpenapiPath = Join-Path -Path $scriptPath -ChildPath "openapi.yaml" +if (Test-Path $targetOpenapiPath) { + Remove-Item $targetOpenapiPath +} + +if ($descriptionUrl.StartsWith("./")) { + Copy-Item -Path $descriptionUrl -Destination $targetOpenapiPath -Force +} +elseif ($descriptionUrl.StartsWith("http")) { + Invoke-WebRequest -Uri $descriptionUrl -OutFile $targetOpenapiPath +} +else { + Start-Process "$kiotaExec" -ArgumentList "download ${descriptionUrl} --clean-output --output $targetOpenapiPath" -Wait -NoNewWindow +} + +$tmpFolder1 = New-TemporaryDirectory +$tmpFolder2 = New-TemporaryDirectory + +Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --output $tmpFolder1" -Wait -NoNewWindow +Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --output $tmpFolder2" -Wait -NoNewWindow + +# Remove variable output files +Remove-Item (Join-Path -Path $tmpFolder1 -ChildPath "kiota-lock.json") +Remove-Item -Force (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log") +Remove-Item (Join-Path -Path $tmpFolder2 -ChildPath "kiota-lock.json") +Remove-Item -Force (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log") + +# Compare hashes +$HashString1 = (Get-ChildItem $tmpFolder1 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String +Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1)) + +$HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String +Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2)) + +Remove-Item $tmpFolder1 +Remove-Item $tmpFolder2 + +if ($HashString1 -eq $HashString2) { + Write-Output "The content of the folders is identical" + Exit 0 +} +else { + Write-Error "The content of the folders is NOT identical" + Exit 1 +} From 173f256115acd8827d247d3d8d18324c82bbf5e4 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 11:57:58 +0200 Subject: [PATCH 02/30] minor --- it/compare-generation.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 3201cf8a01..d522027066 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -76,8 +76,8 @@ Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1)) $HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2)) -Remove-Item $tmpFolder1 -Remove-Item $tmpFolder2 +Remove-Item $tmpFolder1 -Force -Recurse +Remove-Item $tmpFolder2 -Force -Recurse if ($HashString1 -eq $HashString2) { Write-Output "The content of the folders is identical" From 04df20e16f3e1bae18abe27f86d2b2b68f8b4581 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 12:12:05 +0200 Subject: [PATCH 03/30] verify idempotency --- .github/workflows/integration-tests.yml | 179 ++++++++++++++---------- it/compare-generation.ps1 | 21 ++- 2 files changed, 121 insertions(+), 79 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index defb6fe897..35597dc38a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,7 +50,111 @@ jobs: with: name: generator path: publish - integration: + # integration: + # runs-on: ubuntu-latest + # needs: build + # strategy: + # fail-fast: false + # matrix: + # language: + # - java + # - csharp + # - go + # - typescript + # - ruby + # - php + # - python + # description: + # - "./tests/Kiota.Builder.IntegrationTests/InheritingErrors.yaml" + # - "./tests/Kiota.Builder.IntegrationTests/NoUnderscoresInModel.yaml" + # - "oas::petstore" + # - "apisguru::twitter.com:current" + # - "apisguru::notion.com" + # - "apisguru::stripe.com" + # - "https://raw.githubusercontent.com/googlemaps/openapi-specification/main/dist/google-maps-platform-openapi3.yml" + # - "apisguru::meraki.com" + # - "https://developers.pipedrive.com/docs/api/v1/openapi.yaml" + # - "apisguru::twilio.com:api" + # - "apisguru::docusign.net" + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/download-artifact@v3 + # with: + # name: generator + # path: publish + # - run: chmod a+x ./publish/kiota + # # Common dependency needed to run MockServer + # - name: Set up JDK 17 + # uses: actions/setup-java@v3 + # with: + # java-version: "17" + # distribution: "temurin" + # cache: "maven" + + # - name: Setup .NET + # if: matrix.language == 'csharp' + # uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: 7.0.x + # - name: Setup Go + # if: matrix.language == 'go' + # uses: actions/setup-go@v4 + # with: + # go-version: "^1.20" + # - name: Setup Typescript + # if: matrix.language == 'typescript' + # uses: actions/setup-node@v3 + # with: + # node-version: "18" + # - name: Setup Ruby + # if: matrix.language == 'ruby' + # uses: ruby/setup-ruby@v1 + # with: + # ruby-version: "3.2.0" + # bundler-cache: true + # - name: Setup PHP + # if: matrix.language == 'php' + # uses: shivammathur/setup-php@v2 + # with: + # php-version: "8.2" + # coverage: xdebug + # - name: Setup Python + # if: matrix.language == 'python' + # uses: actions/setup-python@v4 + # with: + # python-version: "3.11" + + # - name: Check if test is suppressed + # id: check-suppressed + # run: | + # $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT + # shell: pwsh + # - name: Generate Code + # shell: pwsh + # run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + # - name: Execute the IT test + # shell: pwsh + # run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + # - name: Cleanup the IT folder + # shell: pwsh + # run: ./it/do-clean.ps1 -language ${{ matrix.language }} + # if: always() + + # - id: replace_url + # if: always() + # run: | + # ORIGINAL="${{ matrix.description }}" + # REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" + # echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT + # - uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} + # path: it/${{ matrix.language }} + idempotency: runs-on: ubuntu-latest needs: build strategy: @@ -83,82 +187,9 @@ jobs: name: generator path: publish - run: chmod a+x ./publish/kiota - # Common dependency needed to run MockServer - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: "17" - distribution: "temurin" - cache: "maven" - - # - name: Setup .NET - # if: matrix.language == 'csharp' - # uses: actions/setup-dotnet@v3 - # with: - # dotnet-version: 7.0.x - # - name: Setup Go - # if: matrix.language == 'go' - # uses: actions/setup-go@v4 - # with: - # go-version: "^1.20" - # - name: Setup Typescript - # if: matrix.language == 'typescript' - # uses: actions/setup-node@v3 - # with: - # node-version: "18" - # - name: Setup Ruby - # if: matrix.language == 'ruby' - # uses: ruby/setup-ruby@v1 - # with: - # ruby-version: "3.2.0" - # bundler-cache: true - # - name: Setup PHP - # if: matrix.language == 'php' - # uses: shivammathur/setup-php@v2 - # with: - # php-version: "8.2" - # coverage: xdebug - # - name: Setup Python - # if: matrix.language == 'python' - # uses: actions/setup-python@v4 - # with: - # python-version: "3.11" - - # - name: Check if test is suppressed - # id: check-suppressed - # run: | - # $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT - # shell: pwsh - # - name: Generate Code - # shell: pwsh - # run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - # - name: Execute the IT test - # shell: pwsh - # run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - # - name: Cleanup the IT folder - # shell: pwsh - # run: ./it/do-clean.ps1 -language ${{ matrix.language }} - # if: always() - - name: Verify idempotency shell: pwsh run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - if: always() - - - id: replace_url - if: always() - run: | - ORIGINAL="${{ matrix.description }}" - REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" - echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v3 - if: always() - with: - name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} - path: it/${{ matrix.language }} cleanup: runs-on: ubuntu-latest diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index d522027066..6a2cfb4750 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -3,7 +3,8 @@ param( [Parameter(Mandatory = $true)][string]$descriptionUrl, [Parameter(Mandatory = $true)][string]$language, - [Parameter(Mandatory = $false)][switch]$dev + [Parameter(Mandatory = $false)][switch]$dev, + [Parameter(Mandatory = $false)][switch]$preserveOutput ) if ([string]::IsNullOrEmpty($descriptionUrl)) { @@ -65,9 +66,13 @@ Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${l # Remove variable output files Remove-Item (Join-Path -Path $tmpFolder1 -ChildPath "kiota-lock.json") -Remove-Item -Force (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log") +if (Test-Path (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log")) { + Remove-Item -Force (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log") +} Remove-Item (Join-Path -Path $tmpFolder2 -ChildPath "kiota-lock.json") -Remove-Item -Force (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log") +if (Test-Path (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log")) { + Remove-Item -Force (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log") +} # Compare hashes $HashString1 = (Get-ChildItem $tmpFolder1 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String @@ -76,8 +81,14 @@ Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1)) $HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2)) -Remove-Item $tmpFolder1 -Force -Recurse -Remove-Item $tmpFolder2 -Force -Recurse +if (!$preserveOutput) { + Remove-Item $tmpFolder1 -Force -Recurse + Remove-Item $tmpFolder2 -Force -Recurse +} +else { + Write-Output "Folder 1: $tmpFolder1" + Write-Output "Folder 2: $tmpFolder2" +} if ($HashString1 -eq $HashString2) { Write-Output "The content of the folders is identical" From fd0786737cc78652a0a7484d7a505dfc0c0226ae Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 12:13:01 +0200 Subject: [PATCH 04/30] more --- .github/workflows/integration-tests.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 35597dc38a..07f70d927e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -154,6 +154,16 @@ jobs: # with: # name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} # path: it/${{ matrix.language }} + + # cleanup: + # runs-on: ubuntu-latest + # needs: [integration] + # steps: + # - uses: jimschubert/delete-artifacts-action@v1 + # with: + # artifact_name: "generator" + # min_bytes: "0" + idempotency: runs-on: ubuntu-latest needs: build @@ -190,12 +200,3 @@ jobs: - name: Verify idempotency shell: pwsh run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - - cleanup: - runs-on: ubuntu-latest - needs: [integration] - steps: - - uses: jimschubert/delete-artifacts-action@v1 - with: - artifact_name: "generator" - min_bytes: "0" From 27dd9cdd21f6dd81bcd4fab6b6a13de35ddaf7c1 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 12:35:00 +0200 Subject: [PATCH 05/30] cleanup --- .github/workflows/integration-tests.yml | 265 ++++++++++++------------ it/compare-generation.ps1 | 2 +- 2 files changed, 133 insertions(+), 134 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 07f70d927e..49726321d6 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,31 +1,30 @@ name: integration-tests on: + workflow_dispatch: push: - # workflow_dispatch: - # push: - # branches: [main] - # paths-ignore: - # [ - # "abstractions/**", - # "authentication/**", - # "serialization/**", - # "http/**", - # "**.md", - # ".vscode/**", - # "**.svg", - # ] - # pull_request: - # paths-ignore: - # [ - # "abstractions/**", - # "authentication/**", - # "serialization/**", - # "http/**", - # "**.md", - # ".vscode/**", - # "**.svg", - # ] + branches: [main] + paths-ignore: + [ + "abstractions/**", + "authentication/**", + "serialization/**", + "http/**", + "**.md", + ".vscode/**", + "**.svg", + ] + pull_request: + paths-ignore: + [ + "abstractions/**", + "authentication/**", + "serialization/**", + "http/**", + "**.md", + ".vscode/**", + "**.svg", + ] concurrency: # Only run once for latest commit per ref and cancel other (previous) runs. @@ -50,119 +49,119 @@ jobs: with: name: generator path: publish - # integration: - # runs-on: ubuntu-latest - # needs: build - # strategy: - # fail-fast: false - # matrix: - # language: - # - java - # - csharp - # - go - # - typescript - # - ruby - # - php - # - python - # description: - # - "./tests/Kiota.Builder.IntegrationTests/InheritingErrors.yaml" - # - "./tests/Kiota.Builder.IntegrationTests/NoUnderscoresInModel.yaml" - # - "oas::petstore" - # - "apisguru::twitter.com:current" - # - "apisguru::notion.com" - # - "apisguru::stripe.com" - # - "https://raw.githubusercontent.com/googlemaps/openapi-specification/main/dist/google-maps-platform-openapi3.yml" - # - "apisguru::meraki.com" - # - "https://developers.pipedrive.com/docs/api/v1/openapi.yaml" - # - "apisguru::twilio.com:api" - # - "apisguru::docusign.net" - # steps: - # - uses: actions/checkout@v3 - # - uses: actions/download-artifact@v3 - # with: - # name: generator - # path: publish - # - run: chmod a+x ./publish/kiota - # # Common dependency needed to run MockServer - # - name: Set up JDK 17 - # uses: actions/setup-java@v3 - # with: - # java-version: "17" - # distribution: "temurin" - # cache: "maven" + integration: + runs-on: ubuntu-latest + needs: build + strategy: + fail-fast: false + matrix: + language: + - java + - csharp + - go + - typescript + - ruby + - php + - python + description: + - "./tests/Kiota.Builder.IntegrationTests/InheritingErrors.yaml" + - "./tests/Kiota.Builder.IntegrationTests/NoUnderscoresInModel.yaml" + - "oas::petstore" + - "apisguru::twitter.com:current" + - "apisguru::notion.com" + - "apisguru::stripe.com" + - "https://raw.githubusercontent.com/googlemaps/openapi-specification/main/dist/google-maps-platform-openapi3.yml" + - "apisguru::meraki.com" + - "https://developers.pipedrive.com/docs/api/v1/openapi.yaml" + - "apisguru::twilio.com:api" + - "apisguru::docusign.net" + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: generator + path: publish + - run: chmod a+x ./publish/kiota + # Common dependency needed to run MockServer + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: "17" + distribution: "temurin" + cache: "maven" - # - name: Setup .NET - # if: matrix.language == 'csharp' - # uses: actions/setup-dotnet@v3 - # with: - # dotnet-version: 7.0.x - # - name: Setup Go - # if: matrix.language == 'go' - # uses: actions/setup-go@v4 - # with: - # go-version: "^1.20" - # - name: Setup Typescript - # if: matrix.language == 'typescript' - # uses: actions/setup-node@v3 - # with: - # node-version: "18" - # - name: Setup Ruby - # if: matrix.language == 'ruby' - # uses: ruby/setup-ruby@v1 - # with: - # ruby-version: "3.2.0" - # bundler-cache: true - # - name: Setup PHP - # if: matrix.language == 'php' - # uses: shivammathur/setup-php@v2 - # with: - # php-version: "8.2" - # coverage: xdebug - # - name: Setup Python - # if: matrix.language == 'python' - # uses: actions/setup-python@v4 - # with: - # python-version: "3.11" + - name: Setup .NET + if: matrix.language == 'csharp' + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 7.0.x + - name: Setup Go + if: matrix.language == 'go' + uses: actions/setup-go@v4 + with: + go-version: "^1.20" + - name: Setup Typescript + if: matrix.language == 'typescript' + uses: actions/setup-node@v3 + with: + node-version: "18" + - name: Setup Ruby + if: matrix.language == 'ruby' + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2.0" + bundler-cache: true + - name: Setup PHP + if: matrix.language == 'php' + uses: shivammathur/setup-php@v2 + with: + php-version: "8.2" + coverage: xdebug + - name: Setup Python + if: matrix.language == 'python' + uses: actions/setup-python@v4 + with: + python-version: "3.11" - # - name: Check if test is suppressed - # id: check-suppressed - # run: | - # $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT - # shell: pwsh - # - name: Generate Code - # shell: pwsh - # run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - # - name: Execute the IT test - # shell: pwsh - # run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - # continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - # - name: Cleanup the IT folder - # shell: pwsh - # run: ./it/do-clean.ps1 -language ${{ matrix.language }} - # if: always() + - name: Check if test is suppressed + id: check-suppressed + run: | + $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT + shell: pwsh + - name: Generate Code + shell: pwsh + run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + - name: Execute the IT test + shell: pwsh + run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + - name: Cleanup the IT folder + shell: pwsh + run: ./it/do-clean.ps1 -language ${{ matrix.language }} + if: always() - # - id: replace_url - # if: always() - # run: | - # ORIGINAL="${{ matrix.description }}" - # REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" - # echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT - # - uses: actions/upload-artifact@v3 - # if: always() - # with: - # name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} - # path: it/${{ matrix.language }} + - id: replace_url + if: always() + run: | + ORIGINAL="${{ matrix.description }}" + REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" + echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v3 + if: always() + with: + name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} + path: it/${{ matrix.language }} - # cleanup: - # runs-on: ubuntu-latest - # needs: [integration] - # steps: - # - uses: jimschubert/delete-artifacts-action@v1 - # with: - # artifact_name: "generator" - # min_bytes: "0" + cleanup: + runs-on: ubuntu-latest + needs: [integration] + steps: + - uses: jimschubert/delete-artifacts-action@v1 + with: + artifact_name: "generator" + min_bytes: "0" idempotency: runs-on: ubuntu-latest diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 6a2cfb4750..c28510f0a2 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -86,7 +86,7 @@ if (!$preserveOutput) { Remove-Item $tmpFolder2 -Force -Recurse } else { - Write-Output "Folder 1: $tmpFolder1" + Write-Output "Folder 1: $tmpFolder1" Write-Output "Folder 2: $tmpFolder2" } From e6078638b5a0aca71e79cab22afca099fcc94a87 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 12:54:44 +0200 Subject: [PATCH 06/30] fix --- .github/workflows/integration-tests.yml | 122 ++++++++++++------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 49726321d6..a841f690dc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -90,69 +90,69 @@ jobs: distribution: "temurin" cache: "maven" - - name: Setup .NET - if: matrix.language == 'csharp' - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 7.0.x - - name: Setup Go - if: matrix.language == 'go' - uses: actions/setup-go@v4 - with: - go-version: "^1.20" - - name: Setup Typescript - if: matrix.language == 'typescript' - uses: actions/setup-node@v3 - with: - node-version: "18" - - name: Setup Ruby - if: matrix.language == 'ruby' - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2.0" - bundler-cache: true - - name: Setup PHP - if: matrix.language == 'php' - uses: shivammathur/setup-php@v2 - with: - php-version: "8.2" - coverage: xdebug - - name: Setup Python - if: matrix.language == 'python' - uses: actions/setup-python@v4 - with: - python-version: "3.11" + - name: Setup .NET + if: matrix.language == 'csharp' + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 7.0.x + - name: Setup Go + if: matrix.language == 'go' + uses: actions/setup-go@v4 + with: + go-version: "^1.20" + - name: Setup Typescript + if: matrix.language == 'typescript' + uses: actions/setup-node@v3 + with: + node-version: "18" + - name: Setup Ruby + if: matrix.language == 'ruby' + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2.0" + bundler-cache: true + - name: Setup PHP + if: matrix.language == 'php' + uses: shivammathur/setup-php@v2 + with: + php-version: "8.2" + coverage: xdebug + - name: Setup Python + if: matrix.language == 'python' + uses: actions/setup-python@v4 + with: + python-version: "3.11" - - name: Check if test is suppressed - id: check-suppressed - run: | - $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT - shell: pwsh - - name: Generate Code - shell: pwsh - run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - - name: Execute the IT test - shell: pwsh - run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} - continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - - name: Cleanup the IT folder - shell: pwsh - run: ./it/do-clean.ps1 -language ${{ matrix.language }} - if: always() + - name: Check if test is suppressed + id: check-suppressed + run: | + $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT + shell: pwsh + - name: Generate Code + shell: pwsh + run: ./it/generate-code.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + - name: Execute the IT test + shell: pwsh + run: ./it/exec-cmd.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + - name: Cleanup the IT folder + shell: pwsh + run: ./it/do-clean.ps1 -language ${{ matrix.language }} + if: always() - - id: replace_url - if: always() - run: | - ORIGINAL="${{ matrix.description }}" - REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" - echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v3 - if: always() - with: - name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} - path: it/${{ matrix.language }} + - id: replace_url + if: always() + run: | + ORIGINAL="${{ matrix.description }}" + REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" + echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v3 + if: always() + with: + name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} + path: it/${{ matrix.language }} cleanup: runs-on: ubuntu-latest From 6a857dc5e73646730dfdbb42e9854cd619f88f89 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 15:51:40 +0200 Subject: [PATCH 07/30] upload artifacts on failure --- .github/workflows/integration-tests.yml | 14 ++++++++++++++ .gitignore | 2 ++ it/compare-generation.ps1 | 22 ++++++++++++++-------- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a841f690dc..5f7e94e697 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -199,3 +199,17 @@ jobs: - name: Verify idempotency shell: pwsh run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: idempotency-artifacts + path: it/idempotency-folder*.zip + + cleanup: + runs-on: ubuntu-latest + needs: [integration] + steps: + - uses: jimschubert/delete-artifacts-action@v1 + with: + artifact_name: "generator" + min_bytes: "0" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4601e77b66..72291e08cb 100644 --- a/.gitignore +++ b/.gitignore @@ -448,3 +448,5 @@ reports/ samples/ it/openapi.yaml + +idempotency*.zip diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index c28510f0a2..7c4557a065 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -81,20 +81,26 @@ Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1)) $HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2)) -if (!$preserveOutput) { - Remove-Item $tmpFolder1 -Force -Recurse - Remove-Item $tmpFolder2 -Force -Recurse -} -else { - Write-Output "Folder 1: $tmpFolder1" - Write-Output "Folder 2: $tmpFolder2" -} +Write-Output "Folder 1: $tmpFolder1" +Write-Output "Folder 2: $tmpFolder2" if ($HashString1 -eq $HashString2) { Write-Output "The content of the folders is identical" + + if (!$preserveOutput) { + Remove-Item $tmpFolder1 -Force -Recurse + Remove-Item $tmpFolder2 -Force -Recurse + } + Exit 0 } else { Write-Error "The content of the folders is NOT identical" + + Remove-Item idempotency-folder1.zip -Force + Remove-Item idempotency-folder2.zip -Force + + Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder1.zip + Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder2.zip Exit 1 } From 627e5026e72e9452bc3b2ac09213dcdd8f3ca3b4 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Wed, 16 Aug 2023 16:05:28 +0200 Subject: [PATCH 08/30] fix the CI def --- .github/workflows/integration-tests.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5f7e94e697..7b86f0050b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -204,12 +204,3 @@ jobs: with: name: idempotency-artifacts path: it/idempotency-folder*.zip - - cleanup: - runs-on: ubuntu-latest - needs: [integration] - steps: - - uses: jimschubert/delete-artifacts-action@v1 - with: - artifact_name: "generator" - min_bytes: "0" \ No newline at end of file From 5d586d986c0db3e1ce7fa5cd353bb0914b8a9c1a Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Thu, 17 Aug 2023 10:04:30 +0200 Subject: [PATCH 09/30] fix file upload path --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7b86f0050b..a648cb7a0c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -203,4 +203,4 @@ jobs: if: failure() with: name: idempotency-artifacts - path: it/idempotency-folder*.zip + path: idempotency-folder*.zip From 222ef45ea836f31a7e9d46520b19f9ed50605051 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 21 Aug 2023 15:43:07 -0400 Subject: [PATCH 10/30] - disables zipping when dev mode for idempotency --- it/compare-generation.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 7c4557a065..db7e10e432 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -100,7 +100,9 @@ else { Remove-Item idempotency-folder1.zip -Force Remove-Item idempotency-folder2.zip -Force - Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder1.zip - Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder2.zip - Exit 1 + if ($dev -eq $false) { + Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder1.zip + Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder2.zip + Exit 1 + } } From c278f27b3a256457e18d54b59880a60675a8400f Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 21 Aug 2023 15:43:33 -0400 Subject: [PATCH 11/30] - fixes a bug where the comment for composed types wrapper would be undeterministic --- src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs index 5a70b132dd..18e6c7ddd3 100644 --- a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs +++ b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs @@ -468,7 +468,7 @@ private static CodeTypeBase ConvertComposedTypeToWrapper(CodeClass codeClass, Co ArgumentNullException.ThrowIfNull(codeComposedType); CodeClass newClass; var description = - $"Composed type wrapper for classes {codeComposedType.Types.Select(static x => x.Name).Aggregate(static (x, y) => x + ", " + y)}"; + $"Composed type wrapper for classes {codeComposedType.Types.Select(static x => x.Name).Order(StringComparer.OrdinalIgnoreCase).Aggregate(static (x, y) => x + ", " + y)}"; if (!supportsInnerClasses) { var @namespace = codeClass.GetImmediateParentOfType(); From a561ee65bd9dad3d5f66523d8ee47d6eb3281b2b Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 21 Aug 2023 16:06:47 -0400 Subject: [PATCH 12/30] - fixes an issue where overload indexer would fail replacement Signed-off-by: Vincent Biret --- src/Kiota.Builder/CodeDOM/CodeIndexer.cs | 7 +++++++ src/Kiota.Builder/KiotaBuilder.cs | 1 + .../Refiners/CommonLanguageRefiner.cs | 14 ++++---------- .../Refiners/GoLanguageRefinerTests.cs | 3 ++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Kiota.Builder/CodeDOM/CodeIndexer.cs b/src/Kiota.Builder/CodeDOM/CodeIndexer.cs index d9c801406b..5827876743 100644 --- a/src/Kiota.Builder/CodeDOM/CodeIndexer.cs +++ b/src/Kiota.Builder/CodeDOM/CodeIndexer.cs @@ -39,6 +39,12 @@ public DeprecationInformation? Deprecation { get; set; } + //TODO remove property for v2 + public bool IsLegacyIndexer + { + get; set; + } + public object Clone() { return new CodeIndexer @@ -50,6 +56,7 @@ public object Clone() PathSegment = PathSegment, Deprecation = Deprecation == null ? null : Deprecation with { }, IndexParameter = (CodeParameter)IndexParameter.Clone(), + IsLegacyIndexer = IsLegacyIndexer, }; } } diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 5091cd1784..051846aec6 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -1078,6 +1078,7 @@ private CodeIndexer[] CreateIndexer(string childIdentifier, string childType, Co backCompatibleValue.Name += "-string"; backCompatibleValue.IndexParameter.Type = DefaultIndexerParameterType; backCompatibleValue.Deprecation = new DeprecationInformation("This indexer is deprecated and will be removed in the next major version. Use the one with the typed parameter instead."); + backCompatibleValue.IsLegacyIndexer = true; result.Add(backCompatibleValue); } diff --git a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs index 18e6c7ddd3..debe688d59 100644 --- a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs +++ b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs @@ -596,8 +596,6 @@ protected static void MoveClassesWithNamespaceNamesUnderNamespace(CodeElement cu } CrawlTree(currentElement, MoveClassesWithNamespaceNamesUnderNamespace); } - private static readonly Func IsIndexerTypeSpecificVersion = - (currentIndexerParameterName, existingIndexer) => currentIndexerParameterName.Equals(existingIndexer.IndexParameter.Name, StringComparison.OrdinalIgnoreCase) && !"string".Equals(existingIndexer.IndexParameter.Type.Name, StringComparison.OrdinalIgnoreCase); protected static void ReplaceIndexersByMethodsWithParameter(CodeElement currentElement, bool parameterNullable, Func methodNameCallback, Func parameterNameCallback, GenerationLanguage language) { if (currentElement is CodeIndexer currentIndexer && @@ -606,18 +604,14 @@ protected static void ReplaceIndexersByMethodsWithParameter(CodeElement currentE if (indexerParentClass.ContainsMember(currentElement.Name)) // TODO remove condition for v2 necessary because of the second case of Go block indexerParentClass.RemoveChildElement(currentElement); //TODO remove who block except for last else if body for v2 - var isIndexerStringBackwardCompatible = "string".Equals(currentIndexer.IndexParameter.Type.Name, StringComparison.OrdinalIgnoreCase) && - currentIndexer.Deprecation is not null && currentIndexer.Deprecation.IsDeprecated && - (indexerParentClass.Methods.Any(x => x.IsOfKind(CodeMethodKind.IndexerBackwardCompatibility) && x.OriginalIndexer is not null && IsIndexerTypeSpecificVersion(currentIndexer.IndexParameter.Name, x.OriginalIndexer)) || - (indexerParentClass.Indexer != null && indexerParentClass.Indexer != currentIndexer && IsIndexerTypeSpecificVersion(currentIndexer.IndexParameter.Name, indexerParentClass.Indexer))); - if (isIndexerStringBackwardCompatible && language == GenerationLanguage.Go) + if (currentIndexer.IsLegacyIndexer && language == GenerationLanguage.Go) { - if (indexerParentClass.Methods.FirstOrDefault(x => x.IsOfKind(CodeMethodKind.IndexerBackwardCompatibility) && x.OriginalIndexer is not null && IsIndexerTypeSpecificVersion(currentIndexer.IndexParameter.Name, x.OriginalIndexer)) is CodeMethod typeSpecificCompatibleMethod && + if (indexerParentClass.Methods.FirstOrDefault(x => x.IsOfKind(CodeMethodKind.IndexerBackwardCompatibility) && x.OriginalIndexer is not null && !x.OriginalIndexer.IsLegacyIndexer) is CodeMethod typeSpecificCompatibleMethod && typeSpecificCompatibleMethod.OriginalIndexer is not null) { indexerParentClass.RenameChildElement(typeSpecificCompatibleMethod.Name, typeSpecificCompatibleMethod.Name + typeSpecificCompatibleMethod.OriginalIndexer.IndexParameter.Type.Name.ToFirstCharacterUpperCase()); } - else if (indexerParentClass.Indexer != null && indexerParentClass.Indexer != currentIndexer && IsIndexerTypeSpecificVersion(currentIndexer.IndexParameter.Name, indexerParentClass.Indexer)) + else if (indexerParentClass.Indexer != null && indexerParentClass.Indexer != currentIndexer && !indexerParentClass.Indexer.IsLegacyIndexer) { var specificIndexer = indexerParentClass.Indexer; indexerParentClass.RemoveChildElement(specificIndexer); @@ -625,7 +619,7 @@ currentIndexer.Deprecation is not null && currentIndexer.Deprecation.IsDeprecate } indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); } - else if (!isIndexerStringBackwardCompatible) + else if (!currentIndexer.IsLegacyIndexer) indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); } diff --git a/tests/Kiota.Builder.Tests/Refiners/GoLanguageRefinerTests.cs b/tests/Kiota.Builder.Tests/Refiners/GoLanguageRefinerTests.cs index 40162d4d52..3a30772b85 100644 --- a/tests/Kiota.Builder.Tests/Refiners/GoLanguageRefinerTests.cs +++ b/tests/Kiota.Builder.Tests/Refiners/GoLanguageRefinerTests.cs @@ -408,7 +408,8 @@ public async Task SupportsTypeSpecificOverrideIndexers() Name = "string" }, }, - Deprecation = new("foo") + Deprecation = new("foo"), + IsLegacyIndexer = true }; var typeSpecificIndexer = new CodeIndexer { From 233bb1ba0eb8c50b80fc77ab9693cfad682bb5a3 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 21 Aug 2023 16:20:28 -0400 Subject: [PATCH 13/30] - fixes a bug where description properties could collide with structural properties Signed-off-by: Vincent Biret --- src/Kiota.Builder/KiotaBuilder.cs | 3 +++ ...StructuralPropertiesResevedNameProvider.cs | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 051846aec6..dca34d12c7 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -1084,10 +1084,13 @@ private CodeIndexer[] CreateIndexer(string childIdentifier, string childType, Co return result.ToArray(); } + private static readonly StructuralPropertiesReservedNameProvider structuralPropertiesReservedNameProvider = new(); private CodeProperty? CreateProperty(string childIdentifier, string childType, OpenApiSchema? propertySchema = null, CodeTypeBase? existingType = null, CodePropertyKind kind = CodePropertyKind.Custom) { var propertyName = childIdentifier.CleanupSymbolName(); + if (structuralPropertiesReservedNameProvider.ReservedNames.Contains(propertyName)) + propertyName += "Property"; var resultType = existingType ?? GetPrimitiveType(propertySchema, childType); if (resultType == null) return null; var prop = new CodeProperty diff --git a/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs b/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs new file mode 100644 index 0000000000..89694ea363 --- /dev/null +++ b/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using Kiota.Builder.Refiners; + +namespace Kiota.Builder; +public class StructuralPropertiesReservedNameProvider : IReservedNamesProvider +{ + private readonly Lazy> _reservedNames = new(static () => new(StringComparer.OrdinalIgnoreCase) + { + "GetFieldDeserializers", + "get_field_deserializers", + "Serialize", + "AdditionalData", + "additional_data", + "BackingStore", + "backing_store", + }); + public HashSet ReservedNames => _reservedNames.Value; +} From 33f6c2061bd4041a3525a3b86ebb1e0f3f8684b5 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 21 Aug 2023 16:23:51 -0400 Subject: [PATCH 14/30] - fixes formatting --- src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs b/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs index 89694ea363..3f1d2967dd 100644 --- a/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs +++ b/src/Kiota.Builder/StructuralPropertiesResevedNameProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using Kiota.Builder.Refiners; From 2aa8094bce3d153619a365519e7e667ef64b94a6 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 22 Aug 2023 14:42:52 -0400 Subject: [PATCH 15/30] - fixes an issue where empty properties in model would fail to generate Signed-off-by: Vincent Biret --- src/Kiota.Builder/KiotaBuilder.cs | 42 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index dca34d12c7..90f7f25a65 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -1997,26 +1997,28 @@ private void CreatePropertiesForModelClass(OpenApiUrlTreeNode currentNode, OpenA { if (CollectAllProperties(schema) is var properties && properties.Any()) { - model.AddProperty(properties - .Select(x => - { - var propertySchema = x.Value; - var className = propertySchema.GetSchemaName().CleanupSymbolName(); - if (string.IsNullOrEmpty(className)) - className = $"{model.Name}_{x.Key.CleanupSymbolName()}"; - var shortestNamespaceName = GetModelsNamespaceNameFromReferenceId(propertySchema.Reference?.Id); - var targetNamespace = string.IsNullOrEmpty(shortestNamespaceName) ? ns : - rootNamespace?.FindOrAddNamespace(shortestNamespaceName) ?? ns; - var definition = CreateModelDeclarations(currentNode, propertySchema, default, targetNamespace, string.Empty, typeNameForInlineSchema: className); - if (definition == null) - { - logger.LogWarning("Omitted property {PropertyName} for model {ModelName} in API path {ApiPath}, the schema is invalid.", x.Key, model.Name, currentNode.Path); - return null; - } - return CreateProperty(x.Key, definition.Name, propertySchema: propertySchema, existingType: definition); - }) - .OfType() - .ToArray()); + var propertiesToAdd = properties + .Select(x => + { + var propertySchema = x.Value; + var className = propertySchema.GetSchemaName().CleanupSymbolName(); + if (string.IsNullOrEmpty(className)) + className = $"{model.Name}_{x.Key.CleanupSymbolName()}"; + var shortestNamespaceName = GetModelsNamespaceNameFromReferenceId(propertySchema.Reference?.Id); + var targetNamespace = string.IsNullOrEmpty(shortestNamespaceName) ? ns : + rootNamespace?.FindOrAddNamespace(shortestNamespaceName) ?? ns; + var definition = CreateModelDeclarations(currentNode, propertySchema, default, targetNamespace, string.Empty, typeNameForInlineSchema: className); + if (definition == null) + { + logger.LogWarning("Omitted property {PropertyName} for model {ModelName} in API path {ApiPath}, the schema is invalid.", x.Key, model.Name, currentNode.Path); + return null; + } + return CreateProperty(x.Key, definition.Name, propertySchema: propertySchema, existingType: definition); + }) + .OfType() + .ToArray(); + if (propertiesToAdd.Any()) + model.AddProperty(propertiesToAdd); } } private Dictionary CollectAllProperties(OpenApiSchema schema) From 1a0cb93aa872d3fcc1ac891a633fe1f113792be8 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 22 Aug 2023 15:01:03 -0400 Subject: [PATCH 16/30] - disables validation rules for compare to reduce verbosity Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index db7e10e432..704dfeb586 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -61,8 +61,8 @@ else { $tmpFolder1 = New-TemporaryDirectory $tmpFolder2 = New-TemporaryDirectory -Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --output $tmpFolder1" -Wait -NoNewWindow -Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --output $tmpFolder2" -Wait -NoNewWindow +Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --dvr all --output $tmpFolder1" -Wait -NoNewWindow +Start-Process "$kiotaExec" -ArgumentList "generate --clean-output --language ${language} --openapi ${targetOpenapiPath} --dvr all --output $tmpFolder2" -Wait -NoNewWindow # Remove variable output files Remove-Item (Join-Path -Path $tmpFolder1 -ChildPath "kiota-lock.json") From dbf0b657bf77fc5512356eb439f7c13f69365a58 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 22 Aug 2023 15:30:37 -0400 Subject: [PATCH 17/30] - fixes noisy zip cleanup Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 704dfeb586..5c63ed429f 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -97,12 +97,16 @@ if ($HashString1 -eq $HashString2) { else { Write-Error "The content of the folders is NOT identical" - Remove-Item idempotency-folder1.zip -Force - Remove-Item idempotency-folder2.zip -Force + if (Test-Path "idempotency-folder1.zip") { + Remove-Item "idempotency-folder1.zip" -Force + } + if (Test-Path "idempotency-folder2.zip") { + Remove-Item "idempotency-folder2.zip" -Force + } if ($dev -eq $false) { - Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder1.zip - Compress-Archive -Path $tmpFolder1 -DestinationPath idempotency-folder2.zip + Compress-Archive -Path $tmpFolder1 -DestinationPath "idempotency-folder1.zip" + Compress-Archive -Path $tmpFolder1 -DestinationPath "idempotency-folder2.zip" Exit 1 } } From accaca0c4bf62783cab31a5a2b47affaccab4874 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 22 Aug 2023 16:10:39 -0400 Subject: [PATCH 18/30] - fixes an issue where path extension would be ignored when projecting request builders leading to collisions Signed-off-by: Vincent Biret --- .../OpenApiUrlTreeNodeExtensions.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Kiota.Builder/Extensions/OpenApiUrlTreeNodeExtensions.cs b/src/Kiota.Builder/Extensions/OpenApiUrlTreeNodeExtensions.cs index 13562f876f..41c4ed71c6 100644 --- a/src/Kiota.Builder/Extensions/OpenApiUrlTreeNodeExtensions.cs +++ b/src/Kiota.Builder/Extensions/OpenApiUrlTreeNodeExtensions.cs @@ -89,13 +89,13 @@ public static string GetClassName(this OpenApiUrlTreeNode currentNode, HashSet structuredMimeTypes, string? suffix = default, string? prefix = default, OpenApiOperation? operation = default, OpenApiResponse? response = default, OpenApiSchema? schema = default, bool requestBody = false) { ArgumentNullException.ThrowIfNull(currentNode); - var result = currentNode.GetSegmentName(structuredMimeTypes, suffix, prefix, operation, response, schema, requestBody, static x => string.Join(string.Empty, x.Select(static (y, idx) => idx == 0 ? y : y.ToFirstCharacterUpperCase()))); + var result = currentNode.GetSegmentName(structuredMimeTypes, suffix, prefix, operation, response, schema, requestBody, static x => string.Join(string.Empty, x.Select(static (y, idx) => idx == 0 ? y : y.ToFirstCharacterUpperCase())), false); if (httpVerbs.Contains(result)) return $"{result}Path"; // we don't run the change of an operation conflicting with a path on the same request builder return result; } private static readonly HashSet httpVerbs = new(StringComparer.OrdinalIgnoreCase) { "get", "post", "put", "patch", "delete", "head", "options", "trace" }; - private static string GetSegmentName(this OpenApiUrlTreeNode currentNode, HashSet structuredMimeTypes, string? suffix, string? prefix, OpenApiOperation? operation, OpenApiResponse? response, OpenApiSchema? schema, bool requestBody, Func, string> segmentsReducer) + private static string GetSegmentName(this OpenApiUrlTreeNode currentNode, HashSet structuredMimeTypes, string? suffix, string? prefix, OpenApiOperation? operation, OpenApiResponse? response, OpenApiSchema? schema, bool requestBody, Func, string> segmentsReducer, bool skipExtension = true) { var referenceName = schema?.Reference?.GetClassName(); var rawClassName = referenceName is not null && !string.IsNullOrEmpty(referenceName) ? @@ -107,7 +107,7 @@ private static string GetSegmentName(this OpenApiUrlTreeNode currentNode, HashSe CleanupParametersFromPath(currentNode.Segment)?.ReplaceValueIdentifier())); if (!string.IsNullOrEmpty(rawClassName) && string.IsNullOrEmpty(referenceName)) { - if (stripExtensionForIndexersRegex.IsMatch(rawClassName)) + if (stripExtensionForIndexersTestRegex.IsMatch(rawClassName)) rawClassName = stripExtensionForIndexersRegex.Replace(rawClassName, string.Empty); if ((currentNode?.DoesNodeBelongToItemSubnamespace() ?? false) && idClassNameCleanup.IsMatch(rawClassName)) { @@ -124,7 +124,7 @@ private static string GetSegmentName(this OpenApiUrlTreeNode currentNode, HashSe var classNameSegments = rawClassName?.Split('.', StringSplitOptions.RemoveEmptyEntries).AsEnumerable() ?? Enumerable.Empty(); // only apply the exceptions if we had multiple segments. // Otherwise a single segment class name like `Json` will be returned as an empty string. - if (classNameSegments.Count() > 1) + if (skipExtension && classNameSegments.Count() > 1) classNameSegments = classNameSegments.Except(SegmentsToSkipForClassNames, StringComparer.OrdinalIgnoreCase); return (prefix + segmentsReducer(classNameSegments) + suffix).CleanupSymbolName(); @@ -139,14 +139,17 @@ private static string GetSegmentName(this OpenApiUrlTreeNode currentNode, HashSe }; private static readonly Regex descriptionCleanupRegex = new(@"[\r\n\t]", RegexOptions.Compiled, Constants.DefaultRegexTimeout); public static string CleanupDescription(this string? description) => string.IsNullOrEmpty(description) ? string.Empty : descriptionCleanupRegex.Replace(description, string.Empty); - public static string GetPathItemDescription(this OpenApiUrlTreeNode currentNode, string label, string? defaultValue = default) => - !string.IsNullOrEmpty(label) && (currentNode?.PathItems.ContainsKey(label) ?? false) ? - (currentNode.PathItems[label].Description is string description && !string.IsNullOrEmpty(description) ? - description : - (currentNode.PathItems[label].Summary is string summary && !string.IsNullOrEmpty(summary) ? - summary : - defaultValue)).CleanupDescription() : - (defaultValue ?? string.Empty); + public static string GetPathItemDescription(this OpenApiUrlTreeNode currentNode, string label, string? defaultValue = default) + { + if (currentNode != null && !string.IsNullOrEmpty(label) && currentNode.PathItems.TryGetValue(label, out var pathItem)) + return ((string.IsNullOrEmpty(pathItem.Description), string.IsNullOrEmpty(pathItem.Summary)) switch + { + (false, _) => pathItem.Description, + (_, false) => pathItem.Summary, + (_, _) => defaultValue, + }).CleanupDescription(); + return string.IsNullOrEmpty(defaultValue) ? string.Empty : defaultValue; + } public static bool DoesNodeBelongToItemSubnamespace(this OpenApiUrlTreeNode currentNode) => currentNode.IsPathSegmentWithSingleSimpleParameter(); public static bool IsPathSegmentWithSingleSimpleParameter(this OpenApiUrlTreeNode currentNode) => currentNode?.Segment.IsPathSegmentWithSingleSimpleParameter() ?? false; @@ -166,6 +169,7 @@ private static bool IsPathSegmentWithNumberOfParameters(this string currentSegme return eval(currentSegment.Where(static x => x == RequestParametersChar)); } private static readonly Regex stripExtensionForIndexersRegex = new(@"\.(?:json|yaml|yml|csv|txt)$", RegexOptions.Compiled, Constants.DefaultRegexTimeout); // so {param-name}.json is considered as indexer + private static readonly Regex stripExtensionForIndexersTestRegex = new(@"\{\w+\}\.(?:json|yaml|yml|csv|txt)$", RegexOptions.Compiled, Constants.DefaultRegexTimeout); // so {param-name}.json is considered as indexer public static bool IsComplexPathMultipleParameters(this OpenApiUrlTreeNode currentNode) => (currentNode?.Segment?.IsPathSegmentWithNumberOfParameters(static x => x.Any()) ?? false) && !currentNode.IsPathSegmentWithSingleSimpleParameter(); public static string GetUrlTemplate(this OpenApiUrlTreeNode currentNode) From 70bc8f9436dfbfb0718c8c8b203d5a3efd5b92da Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 09:17:36 -0400 Subject: [PATCH 19/30] - fixes backward compatible indexer method for Go --- src/Kiota.Builder/CodeDOM/CodeClass.cs | 2 +- .../Refiners/CommonLanguageRefiner.cs | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Kiota.Builder/CodeDOM/CodeClass.cs b/src/Kiota.Builder/CodeDOM/CodeClass.cs index 2b4fcdbe7b..7a01892305 100644 --- a/src/Kiota.Builder/CodeDOM/CodeClass.cs +++ b/src/Kiota.Builder/CodeDOM/CodeClass.cs @@ -45,7 +45,7 @@ public CodeComposedTypeBase? OriginalComposedType { get; set; } - public CodeIndexer? Indexer => InnerChildElements.Values.OfType().FirstOrDefault(static x => !x.Deprecation?.IsDeprecated ?? true); + public CodeIndexer? Indexer => InnerChildElements.Values.OfType().FirstOrDefault(static x => !x.IsLegacyIndexer); public void AddIndexer(params CodeIndexer[] indexers) { if (indexers == null || Array.Exists(indexers, static x => x == null)) diff --git a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs index debe688d59..7e708a073c 100644 --- a/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs +++ b/src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs @@ -603,21 +603,27 @@ protected static void ReplaceIndexersByMethodsWithParameter(CodeElement currentE { if (indexerParentClass.ContainsMember(currentElement.Name)) // TODO remove condition for v2 necessary because of the second case of Go block indexerParentClass.RemoveChildElement(currentElement); - //TODO remove who block except for last else if body for v2 - if (currentIndexer.IsLegacyIndexer && language == GenerationLanguage.Go) + //TODO remove whole block except for last else if body for v2 + if (language == GenerationLanguage.Go) { - if (indexerParentClass.Methods.FirstOrDefault(x => x.IsOfKind(CodeMethodKind.IndexerBackwardCompatibility) && x.OriginalIndexer is not null && !x.OriginalIndexer.IsLegacyIndexer) is CodeMethod typeSpecificCompatibleMethod && - typeSpecificCompatibleMethod.OriginalIndexer is not null) + if (currentIndexer.IsLegacyIndexer) { - indexerParentClass.RenameChildElement(typeSpecificCompatibleMethod.Name, typeSpecificCompatibleMethod.Name + typeSpecificCompatibleMethod.OriginalIndexer.IndexParameter.Type.Name.ToFirstCharacterUpperCase()); + if (indexerParentClass.Indexer is CodeIndexer specificIndexer && specificIndexer != currentIndexer && !specificIndexer.IsLegacyIndexer) + { + indexerParentClass.RemoveChildElement(specificIndexer); + indexerParentClass.AddMethod(CodeMethod.FromIndexer(specificIndexer, methodNameCallback, parameterNameCallback, parameterNullable, true)); + } + indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); } - else if (indexerParentClass.Indexer != null && indexerParentClass.Indexer != currentIndexer && !indexerParentClass.Indexer.IsLegacyIndexer) + else { - var specificIndexer = indexerParentClass.Indexer; - indexerParentClass.RemoveChildElement(specificIndexer); - indexerParentClass.AddMethod(CodeMethod.FromIndexer(specificIndexer, methodNameCallback, parameterNameCallback, parameterNullable, true)); + if (indexerParentClass.GetChildElements(true).OfType().FirstOrDefault(static x => x.IsLegacyIndexer) is CodeIndexer legacyIndexer) + { + indexerParentClass.RemoveChildElement(legacyIndexer); + indexerParentClass.AddMethod(CodeMethod.FromIndexer(legacyIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); + } + indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable, true)); } - indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); } else if (!currentIndexer.IsLegacyIndexer) indexerParentClass.AddMethod(CodeMethod.FromIndexer(currentIndexer, methodNameCallback, parameterNameCallback, parameterNullable)); From ec5740c5a9523aa035db3e977278c140efb1ac8b Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 09:54:48 -0400 Subject: [PATCH 20/30] - adds a suppression mechanism for idempotency --- .github/workflows/integration-tests.yml | 7 +++++ it/config.json | 42 +++++++++++++++++++++++++ it/get-is-suppressed.ps1 | 12 +++++-- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a648cb7a0c..1897d03f5e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -196,9 +196,16 @@ jobs: name: generator path: publish - run: chmod a+x ./publish/kiota + - name: Check if test is suppressed + id: check-suppressed + run: | + $isSuppressed = . ./it/get-is-suppressed.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} -kind idempotency + Write-Output "IS_SUPPRESSED=$($isSuppressed.ToString().ToLowerInvariant())" >> $Env:GITHUB_OUTPUT + shell: pwsh - name: Verify idempotency shell: pwsh run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} + continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} - uses: actions/upload-artifact@v3 if: failure() with: diff --git a/it/config.json b/it/config.json index a4b0f69fbd..6671c57895 100644 --- a/it/config.json +++ b/it/config.json @@ -154,6 +154,12 @@ "Language": "python", "Rationale": "https://github.com/microsoft/kiota/issues/2957" } + ], + "IdempotencySuppressions": [ + { + "Language": "typescript", + "Rationale": "https://github.com/microsoft/kiota/issues/1812" + } ] }, "apisguru::twilio.com:api": { @@ -244,6 +250,32 @@ "Language": "ruby", "Rationale": "https://github.com/microsoft/kiota/issues/1816" } + ], + "IdempotencySuppressions": [ + { + "Language": "typescript", + "Rationale": "https://github.com/microsoft/kiota/issues/1812" + }, + { + "Language": "go", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" + }, + { + "Language": "php", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" + }, + { + "Language": "java", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" + }, + { + "Language": "csharp", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" + }, + { + "Language": "ruby", + "Rationale": "https://github.com/microsoft/kiota/issues/1816" + } ] }, "apisguru::meraki.com": { @@ -280,6 +312,16 @@ "Language": "ruby", "Rationale": "https://github.com/microsoft/kiota/issues/1816" } + ], + "IdempotencySuppressions": [ + { + "Language": "php", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" + }, + { + "Language": "typescript", + "Rationale": "https://github.com/microsoft/kiota/issues/1812" + } ] } } diff --git a/it/get-is-suppressed.ps1 b/it/get-is-suppressed.ps1 index 535ea126c0..20079f6556 100755 --- a/it/get-is-suppressed.ps1 +++ b/it/get-is-suppressed.ps1 @@ -2,7 +2,8 @@ param( [Parameter(Mandatory = $true)][string]$descriptionUrl, - [Parameter(Mandatory = $true)][string]$language + [Parameter(Mandatory = $true)][string]$language, + [string]$kind = "integration" ) if ([string]::IsNullOrEmpty($descriptionUrl)) { @@ -21,13 +22,20 @@ $jsonValue = Get-Content -Path $configPath -Raw | ConvertFrom-Json $descriptionValue = $jsonValue.psobject.properties.Where({ $_.name -eq $descriptionUrl }).value if ($null -ne $descriptionValue) { - if ($descriptionValue.PSObject.Properties.Name -contains "Suppressions") { + if ($kind -eq "integration" -and $descriptionValue.PSObject.Properties.Name -contains "Suppressions") { $languageInformation = $descriptionValue.Suppressions | Where-Object { $_.Language -eq $language -or $_.Language -eq "all" } | Select-Object -First 1 if ($null -ne $languageInformation) { Write-Warning "Suppressed $descriptionUrl for $language, rationale: $($languageInformation.Rationale)" return $true } } + elseif ($kind -eq "idempotency" -and $descriptionValue.PSObject.Properties.Name -contains "IdempotencySuppressions") { + $languageInformation = $descriptionValue.IdempotencySuppressions | Where-Object { $_.Language -eq $language -or $_.Language -eq "all" } | Select-Object -First 1 + if ($null -ne $languageInformation) { + Write-Warning "Suppressed $descriptionUrl for $language, rationale: $($languageInformation.Rationale)" + return $true + } + } } Write-Information "No suppression found for $descriptionUrl for $language" From f0e2a175267b89f20d337a2d5a50c9f8013bd834 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 09:58:39 -0400 Subject: [PATCH 21/30] - moves generator cleanup job after idempotency Signed-off-by: Vincent Biret --- .github/workflows/integration-tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1897d03f5e..c3cb6fc4da 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -154,15 +154,6 @@ jobs: name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} path: it/${{ matrix.language }} - cleanup: - runs-on: ubuntu-latest - needs: [integration] - steps: - - uses: jimschubert/delete-artifacts-action@v1 - with: - artifact_name: "generator" - min_bytes: "0" - idempotency: runs-on: ubuntu-latest needs: build @@ -211,3 +202,12 @@ jobs: with: name: idempotency-artifacts path: idempotency-folder*.zip + + cleanup: + runs-on: ubuntu-latest + needs: [integration, idempotency] + steps: + - uses: jimschubert/delete-artifacts-action@v1 + with: + artifact_name: "generator" + min_bytes: "0" From d93777e47912a0d819d44d1b712d9b770b83dbb2 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 09:59:29 -0400 Subject: [PATCH 22/30] - adds changelog entry for idempotency fixes Signed-off-by: Vincent Biret --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e2320fb42..f43f06497d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Multiple fixes to guarantee idempotency of subsequent runs. [#2442](https://github.com/microsoft/kiota/issues/2442) - Fix issue with generating classes with Aliases(PHP)[microsoftgraph/msgraph-beta-sdk-php#197](https://github.com/microsoftgraph/msgraph-beta-sdk-php/pull/197) - Flattens the models namespaces in Ruby to avoid circular dependencies. - Adds ObjectId as a reserved keyword in Ruby to have memory management issues. From 2425aa81ccef7cc43a17600bddadc20b86b5df02 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 11:11:33 -0400 Subject: [PATCH 23/30] - adds missing suppression for ruby Signed-off-by: Vincent Biret --- it/config.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/it/config.json b/it/config.json index 6671c57895..e6914ec3e0 100644 --- a/it/config.json +++ b/it/config.json @@ -222,6 +222,12 @@ "Language": "ruby", "Rationale": "https://github.com/microsoft/kiota/issues/2484" } + ], + "IdempotencySuppressions": [ + { + "Language": "ruby", + "Rationale": "https://github.com/microsoft/kiota/issues/1816" + } ] }, "apisguru::stripe.com": { From 90f3ed52d4e0bf34165761806dc140aaeac07e6d Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 13:02:25 -0400 Subject: [PATCH 24/30] - adds python to strip suppressions Signed-off-by: Vincent Biret --- it/config.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/it/config.json b/it/config.json index e6914ec3e0..44b1e2d3f2 100644 --- a/it/config.json +++ b/it/config.json @@ -281,6 +281,10 @@ { "Language": "ruby", "Rationale": "https://github.com/microsoft/kiota/issues/1816" + }, + { + "Language": "python", + "Rationale": "https://github.com/microsoft/kiota/issues/3067" } ] }, From 675f389e2e2f84ca26399057e2679dea8bf1eed7 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 14:41:20 -0400 Subject: [PATCH 25/30] - disables tutorial mode for integration tests Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 1 + it/generate-code.ps1 | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 5c63ed429f..9810d0b0b3 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -26,6 +26,7 @@ function New-TemporaryDirectory { $scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition $rootPath = Join-Path -Path $scriptPath -ChildPath ".." +$Env:KIOTA_TUTORIAL_ENABLED = "false" $executableName = "kiota" if ($IsWindows) { $executableName = "kiota.exe" diff --git a/it/generate-code.ps1 b/it/generate-code.ps1 index 11274a4270..1c1bada5e9 100755 --- a/it/generate-code.ps1 +++ b/it/generate-code.ps1 @@ -21,13 +21,13 @@ $additionalArgumentCmd = Join-Path -Path $scriptPath -ChildPath "get-additional- $additionalArguments = Invoke-Expression "$additionalArgumentCmd -descriptionUrl $descriptionUrl -language $language" $rootPath = Join-Path -Path $scriptPath -ChildPath ".." +$Env:KIOTA_TUTORIAL_ENABLED = "false" $executableName = "kiota" if ($IsWindows) { $executableName = "kiota.exe" } -switch ($dev) -{ +switch ($dev) { $true { Write-Warning "Using kiota in dev mode" $kiotaExec = Join-Path -Path $rootPath -ChildPath "src" -AdditionalChildPath "kiota", "bin", "Debug", "net7.0", $executableName @@ -46,9 +46,11 @@ if (Test-Path $targetOpenapiPath) { if ($descriptionUrl.StartsWith("./")) { Copy-Item -Path $descriptionUrl -Destination $targetOpenapiPath -Force -} elseif ($descriptionUrl.StartsWith("http")) { +} +elseif ($descriptionUrl.StartsWith("http")) { Invoke-WebRequest -Uri $descriptionUrl -OutFile $targetOpenapiPath -} else { +} +else { Start-Process "$kiotaExec" -ArgumentList "download ${descriptionUrl} --clean-output --output $targetOpenapiPath" -Wait -NoNewWindow } From a48bd9fd82f6e164467f5459e9a51c05f5a10ff0 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 14:56:30 -0400 Subject: [PATCH 26/30] - attempt at fixing failed idempotency uploads Signed-off-by: Vincent Biret --- .github/workflows/integration-tests.yml | 10 ++++++++-- it/compare-generation.ps1 | 14 ++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c3cb6fc4da..42b101d8c7 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -197,10 +197,16 @@ jobs: shell: pwsh run: ./it/compare-generation.ps1 -descriptionUrl ${{ matrix.description }} -language ${{ matrix.language }} continue-on-error: ${{ steps.check-suppressed.outputs.IS_SUPPRESSED == 'true' }} + - id: replace_url + if: always() + run: | + ORIGINAL="${{ matrix.description }}" + REPLACED="${ORIGINAL//[-:<>|\*\?\\\/\.]/_}" + echo "ARTKEY=$REPLACED" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@v3 - if: failure() + if: always() with: - name: idempotency-artifacts + name: idempotency-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} path: idempotency-folder*.zip cleanup: diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 9810d0b0b3..5bec845dc5 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -97,17 +97,19 @@ if ($HashString1 -eq $HashString2) { } else { Write-Error "The content of the folders is NOT identical" + $archivePath1 = Join-Path $rootPath -ChildPath "idempotency-folder1.zip" + $archivePath2 = Join-Path $rootPath -ChildPath "idempotency-folder2.zip" - if (Test-Path "idempotency-folder1.zip") { - Remove-Item "idempotency-folder1.zip" -Force + if (Test-Path $archivePath1) { + Remove-Item $archivePath1 -Force } - if (Test-Path "idempotency-folder2.zip") { - Remove-Item "idempotency-folder2.zip" -Force + if (Test-Path $archivePath2) { + Remove-Item $archivePath2 -Force } if ($dev -eq $false) { - Compress-Archive -Path $tmpFolder1 -DestinationPath "idempotency-folder1.zip" - Compress-Archive -Path $tmpFolder1 -DestinationPath "idempotency-folder2.zip" + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1 + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2 Exit 1 } } From 0088e70613b95f6fe66b9463652526ac624960b3 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 15:24:29 -0400 Subject: [PATCH 27/30] - another attempt at fixing archives upload Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 5bec845dc5..9f64f6ba05 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -101,15 +101,15 @@ else { $archivePath2 = Join-Path $rootPath -ChildPath "idempotency-folder2.zip" if (Test-Path $archivePath1) { - Remove-Item $archivePath1 -Force + Remove-Item $archivePath1 -Force -Verbose } if (Test-Path $archivePath2) { - Remove-Item $archivePath2 -Force + Remove-Item $archivePath2 -Force -Verbose } if ($dev -eq $false) { - Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1 - Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2 - Exit 1 + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1 -Verbose + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2 -Verbose } + Exit 1 } From e970badaf53ca0b117953435bfca796001c6b33c Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 15:33:14 -0400 Subject: [PATCH 28/30] - updates getting the scripts directory Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 5 +-- it/exec-cmd.ps1 | 40 +++++++++---------- it/generate-code.ps1 | 7 ++-- it/get-additional-arguments.ps1 | 3 +- it/get-is-suppressed.ps1 | 3 +- scripts/get-release-notes.ps1 | 5 +-- scripts/get-version-from-csproj.ps1 | 3 +- ...te-version-suffix-for-source-generator.ps1 | 5 +-- scripts/update-versions.ps1 | 5 +-- 9 files changed, 31 insertions(+), 45 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 9f64f6ba05..3122606029 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -23,8 +23,7 @@ function New-TemporaryDirectory { New-Item -ItemType Directory -Path (Join-Path $parent $name) } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$rootPath = Join-Path -Path $scriptPath -ChildPath ".." +$rootPath = Join-Path -Path $PSScriptRoot -ChildPath ".." $Env:KIOTA_TUTORIAL_ENABLED = "false" $executableName = "kiota" @@ -44,7 +43,7 @@ switch ($dev) { } } -$targetOpenapiPath = Join-Path -Path $scriptPath -ChildPath "openapi.yaml" +$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml" if (Test-Path $targetOpenapiPath) { Remove-Item $targetOpenapiPath } diff --git a/it/exec-cmd.ps1 b/it/exec-cmd.ps1 index a8093edec2..e470821fe6 100755 --- a/it/exec-cmd.ps1 +++ b/it/exec-cmd.ps1 @@ -28,19 +28,15 @@ function Invoke-Call { } } -function Retry([Action]$action) -{ - $attempts=10 - $sleepInSeconds=1 - do - { - try - { +function Retry([Action]$action) { + $attempts = 10 + $sleepInSeconds = 1 + do { + try { $action.Invoke(); break; } - catch [Exception] - { + catch [Exception] { Write-Host $_.Exception.Message } $attempts-- @@ -48,18 +44,17 @@ function Retry([Action]$action) } while ($attempts -gt 0) } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$testPath = Join-Path -Path $scriptPath -ChildPath $language -$mockServerPath = Join-Path -Path $scriptPath -ChildPath "mockserver" +$testPath = Join-Path -Path $PSScriptRoot -ChildPath $language +$mockServerPath = Join-Path -Path $PSScriptRoot -ChildPath "mockserver" function Kill-MockServer { Push-Location $mockServerPath - mvn --batch-mode mockserver:stopForked + mvn --batch-mode mockserver:stopForked Pop-Location } $mockSeverITFolder = $null -$configPath = Join-Path -Path $scriptPath -ChildPath "config.json" +$configPath = Join-Path -Path $PSScriptRoot -ChildPath "config.json" $jsonValue = Get-Content -Path $configPath -Raw | ConvertFrom-Json $descriptionValue = $jsonValue.psobject.properties.Where({ $_.name -eq $descriptionUrl }).value if ($null -ne $descriptionValue) { @@ -73,14 +68,14 @@ if (!([string]::IsNullOrEmpty($mockSeverITFolder))) { # Kill any leftover MockServer Kill-MockServer Push-Location $mockServerPath - mvn --batch-mode mockserver:runForked + mvn --batch-mode mockserver:runForked Pop-Location # Provision Mock server with the right spec - $openapiUrl = Join-Path -Path $scriptPath -ChildPath "openapi.yaml" + $openapiUrl = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml" # provision MockServer to mock the specific openapi description https://www.mock-server.com/mock_server/using_openapi.html#button_open_api_filepath - Retry({Invoke-WebRequest -Method PUT -Body "{ `"specUrlOrPayload`": `"$openapiUrl`" }" -Uri http://localhost:1080/mockserver/openapi -ContentType application/json}) + Retry({ Invoke-WebRequest -Method PUT -Body "{ `"specUrlOrPayload`": `"$openapiUrl`" }" -Uri http://localhost:1080/mockserver/openapi -ContentType application/json }) } Push-Location $testPath @@ -101,7 +96,8 @@ if ($language -eq "csharp") { } -ErrorAction Stop Pop-Location - } else { + } + else { Invoke-Call -ScriptBlock { dotnet build } -ErrorAction Stop @@ -124,7 +120,8 @@ elseif ($language -eq "java") { } -ErrorAction Stop Pop-Location - } else { + } + else { Invoke-Call -ScriptBlock { mvn clean compile --batch-mode } -ErrorAction Stop @@ -147,7 +144,8 @@ elseif ($language -eq "go") { } -ErrorAction Stop Pop-Location - } else { + } + else { Invoke-Call -ScriptBlock { go install go build diff --git a/it/generate-code.ps1 b/it/generate-code.ps1 index 1c1bada5e9..94e20f1234 100755 --- a/it/generate-code.ps1 +++ b/it/generate-code.ps1 @@ -16,10 +16,9 @@ if ([string]::IsNullOrEmpty($language)) { exit 1 } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$additionalArgumentCmd = Join-Path -Path $scriptPath -ChildPath "get-additional-arguments.ps1" +$additionalArgumentCmd = Join-Path -Path $PSScriptRoot -ChildPath "get-additional-arguments.ps1" $additionalArguments = Invoke-Expression "$additionalArgumentCmd -descriptionUrl $descriptionUrl -language $language" -$rootPath = Join-Path -Path $scriptPath -ChildPath ".." +$rootPath = Join-Path -Path $PSScriptRoot -ChildPath ".." $Env:KIOTA_TUTORIAL_ENABLED = "false" $executableName = "kiota" @@ -39,7 +38,7 @@ switch ($dev) { } } -$targetOpenapiPath = Join-Path -Path $scriptPath -ChildPath "openapi.yaml" +$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml" if (Test-Path $targetOpenapiPath) { Remove-Item $targetOpenapiPath } diff --git a/it/get-additional-arguments.ps1 b/it/get-additional-arguments.ps1 index 3096b164c9..f13eab2cbe 100755 --- a/it/get-additional-arguments.ps1 +++ b/it/get-additional-arguments.ps1 @@ -38,8 +38,7 @@ elseif ($language -eq "python") { $command = " --output `"./it/$language/integration_test/client`" --namespace-name `"integration_test.client`"" } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$configPath = Join-Path -Path $scriptPath -ChildPath "config.json" +$configPath = Join-Path -Path $PSScriptRoot -ChildPath "config.json" $jsonValue = Get-Content -Path $configPath -Raw | ConvertFrom-Json $descriptionValue = $jsonValue.psobject.properties.Where({ $_.name -eq $descriptionUrl }).value if ($null -ne $descriptionValue) { diff --git a/it/get-is-suppressed.ps1 b/it/get-is-suppressed.ps1 index 20079f6556..3a8e25e507 100755 --- a/it/get-is-suppressed.ps1 +++ b/it/get-is-suppressed.ps1 @@ -16,8 +16,7 @@ if ([string]::IsNullOrEmpty($language)) { exit 1 } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$configPath = Join-Path -Path $scriptPath -ChildPath "config.json" +$configPath = Join-Path -Path $PSScriptRoot -ChildPath "config.json" $jsonValue = Get-Content -Path $configPath -Raw | ConvertFrom-Json $descriptionValue = $jsonValue.psobject.properties.Where({ $_.name -eq $descriptionUrl }).value diff --git a/scripts/get-release-notes.ps1 b/scripts/get-release-notes.ps1 index 36625c37b9..1d4843f92c 100644 --- a/scripts/get-release-notes.ps1 +++ b/scripts/get-release-notes.ps1 @@ -9,10 +9,7 @@ param ( $version = $version.TrimStart("v") -# Get current script directory -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition - -$changelogPath = Join-Path -Path $scriptPath -ChildPath "../CHANGELOG.md" +$changelogPath = Join-Path -Path $PSScriptRoot -ChildPath "../CHANGELOG.md" $changeLogContent = Get-Content $changelogPath -Raw $headerLine = "## [$version]" diff --git a/scripts/get-version-from-csproj.ps1 b/scripts/get-version-from-csproj.ps1 index a06c9c145c..2f6aaea5e9 100644 --- a/scripts/get-version-from-csproj.ps1 +++ b/scripts/get-version-from-csproj.ps1 @@ -3,8 +3,7 @@ param ( [bool] $isGHA ) -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$csprojPath = Join-Path $scriptPath "../src/kiota/kiota.csproj" +$csprojPath = Join-Path $PSScriptRoot "../src/kiota/kiota.csproj" $xml = [Xml] (Get-Content $csprojPath) $version = $xml.Project.PropertyGroup.VersionPrefix[0] Write-Output "csproj version is $version" diff --git a/scripts/update-version-suffix-for-source-generator.ps1 b/scripts/update-version-suffix-for-source-generator.ps1 index b6e88cbf22..082737573e 100644 --- a/scripts/update-version-suffix-for-source-generator.ps1 +++ b/scripts/update-version-suffix-for-source-generator.ps1 @@ -9,9 +9,8 @@ if ($versionSuffix -eq "") { } # Get current script directory -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$appCsProjPath = Join-Path -Path $scriptPath -ChildPath "../src/kiota/kiota.csproj" -$builderCsProjPath = Join-Path -Path $scriptPath -ChildPath "../src/Kiota.Builder/Kiota.Builder.csproj" +$appCsProjPath = Join-Path -Path $PSScriptRoot -ChildPath "../src/kiota/kiota.csproj" +$builderCsProjPath = Join-Path -Path $PSScriptRoot -ChildPath "../src/Kiota.Builder/Kiota.Builder.csproj" #load xml from csproj $appCsProj = [xml](Get-Content $appCsProjPath) diff --git a/scripts/update-versions.ps1 b/scripts/update-versions.ps1 index 8a9c2dc819..df36c6326f 100644 --- a/scripts/update-versions.ps1 +++ b/scripts/update-versions.ps1 @@ -104,11 +104,8 @@ function Retry-Command { } } -# Get current script directory -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition - # Read the appsettings.json file -$mainSettings = Join-Path -Path $scriptPath -ChildPath "..\src\kiota\appsettings.json" +$mainSettings = Join-Path -Path $PSScriptRoot -ChildPath "..\src\kiota\appsettings.json" $appSettings = Get-Content -Path $mainSettings -Raw | ConvertFrom-Json foreach ($languageName in ($appSettings.Languages | Get-Member -MemberType NoteProperty | Select-Object -ExpandProperty Name | Sort-Object)) { From d2b8282382eec4864461960359fa73ba6d1641f1 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 15:35:10 -0400 Subject: [PATCH 29/30] - zips generation results to accelerate workflow Signed-off-by: Vincent Biret --- .github/workflows/integration-tests.yml | 2 +- it/do-clean.ps1 | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 42b101d8c7..83156b4ff1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -152,7 +152,7 @@ jobs: if: always() with: name: generation-results-${{ matrix.language }}-${{ steps.replace_url.outputs.ARTKEY }} - path: it/${{ matrix.language }} + path: it/${{ matrix.language }}.zip idempotency: runs-on: ubuntu-latest diff --git a/it/do-clean.ps1 b/it/do-clean.ps1 index 4b80fee2a8..464017dbfc 100755 --- a/it/do-clean.ps1 +++ b/it/do-clean.ps1 @@ -9,8 +9,7 @@ if ([string]::IsNullOrEmpty($language)) { exit 1 } -$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$testPath = Join-Path -Path $scriptPath -ChildPath $language +$testPath = Join-Path -Path $PSScriptRoot -ChildPath $language Push-Location $testPath if ($language -eq "csharp") { @@ -30,3 +29,9 @@ elseif ($language -eq "php") { Remove-Item composer.lock -ErrorAction SilentlyContinue } Pop-Location + +$archiveLocation = Join-Path -Path $PSScriptRoot -ChildPath "$language.zip" +if (Test-Path $archiveLocation) { + Remove-Item $archiveLocation -Force -ErrorAction SilentlyContinue -Verbose +} +Compress-Archive -Path $testPath -DestinationPath $archiveLocation -Verbose From af214cc2d6c83b03525dd63ff4372bd0b7238f30 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Wed, 23 Aug 2023 15:37:22 -0400 Subject: [PATCH 30/30] - corrects script verbosity Signed-off-by: Vincent Biret --- it/compare-generation.ps1 | 5 +++-- it/do-clean.ps1 | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/it/compare-generation.ps1 b/it/compare-generation.ps1 index 3122606029..c8c40cf710 100755 --- a/it/compare-generation.ps1 +++ b/it/compare-generation.ps1 @@ -107,8 +107,9 @@ else { } if ($dev -eq $false) { - Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1 -Verbose - Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2 -Verbose + Write-Host "Creating archives at location $archivePath1 and $archivePath2" + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath1 + Compress-Archive -Path $tmpFolder1 -DestinationPath $archivePath2 } Exit 1 } diff --git a/it/do-clean.ps1 b/it/do-clean.ps1 index 464017dbfc..923fb4c9ee 100755 --- a/it/do-clean.ps1 +++ b/it/do-clean.ps1 @@ -34,4 +34,4 @@ $archiveLocation = Join-Path -Path $PSScriptRoot -ChildPath "$language.zip" if (Test-Path $archiveLocation) { Remove-Item $archiveLocation -Force -ErrorAction SilentlyContinue -Verbose } -Compress-Archive -Path $testPath -DestinationPath $archiveLocation -Verbose +Compress-Archive -Path $testPath -DestinationPath $archiveLocation