From 635563a8e18489ac2c413276658293694c0e9507 Mon Sep 17 00:00:00 2001 From: "Michelle A. Taal" Date: Wed, 25 Oct 2023 15:32:22 +0200 Subject: [PATCH 1/3] Ignore organisation secrets --- src/Program.cs | 43 ++++++++++++++++++++++-------- src/Properties/launchSettings.json | 2 +- src/SecretsMigrator.csproj | 8 +++--- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/Program.cs b/src/Program.cs index e40250e..7482f7a 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,6 +1,7 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.CommandLine.Parsing; +using System; namespace SecretsMigrator { @@ -39,6 +40,10 @@ public static async Task Main(string[] args) { IsRequired = true }; + var ignoreOrgSecrets = new Option("--ignore-org-secrets") + { + IsRequired = false + }; var verbose = new Option("--verbose") { IsRequired = false @@ -50,14 +55,15 @@ public static async Task Main(string[] args) root.AddOption(targetRepo); root.AddOption(sourcePat); root.AddOption(targetPat); + root.AddOption(ignoreOrgSecrets); root.AddOption(verbose); - root.Handler = CommandHandler.Create(Invoke); + root.Handler = CommandHandler.Create(Invoke); await root.InvokeAsync(args); } - public static async Task Invoke(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string sourcePat, string targetPat, bool verbose = false) + public static async Task Invoke(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string sourcePat, string targetPat, bool ignoreOrgSecrets = false, bool verbose = false) { _log.Verbose = verbose; @@ -67,14 +73,16 @@ public static async Task Invoke(string sourceOrg, string sourceRepo, string targ _log.LogInformation($"TARGET ORG: {targetOrg}"); _log.LogInformation($"TARGET REPO: {targetRepo}"); - var branchName = "migrate-secrets"; - var workflow = GenerateWorkflow(targetOrg, targetRepo, branchName); + var id = (new Random().Next(1000, 9999)).ToString(); + var branchName = $"migrate-secrets-{id}"; + var workflow = GenerateWorkflow(sourceOrg, sourceRepo, targetOrg, targetRepo, branchName, ignoreOrgSecrets); var githubClient = new GithubClient(_log, sourcePat); var githubApi = new GithubApi(githubClient, "https://api.github.com"); var (publicKey, publicKeyId) = await githubApi.GetRepoPublicKey(sourceOrg, sourceRepo); - await githubApi.CreateRepoSecret(sourceOrg, sourceRepo, publicKey, publicKeyId, "SECRETS_MIGRATOR_PAT", targetPat); + await githubApi.CreateRepoSecret(sourceOrg, sourceRepo, publicKey, publicKeyId, "SECRETS_MIGRATOR_TARGET_PAT", targetPat); + await githubApi.CreateRepoSecret(sourceOrg, sourceRepo, publicKey, publicKeyId, "SECRETS_MIGRATOR_SOURCE_PAT", sourcePat); var defaultBranch = await githubApi.GetDefaultBranch(sourceOrg, sourceRepo); var masterCommitSha = await githubApi.GetCommitSha(sourceOrg, sourceRepo, defaultBranch); @@ -85,7 +93,7 @@ public static async Task Invoke(string sourceOrg, string sourceRepo, string targ _log.LogSuccess($"Secrets migration in progress. Check on status at https://github.com/{sourceOrg}/{sourceRepo}/actions"); } - private static string GenerateWorkflow(string targetOrg, string targetRepo, string branchName) + private static string GenerateWorkflow(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string branchName, bool ignoreOrgSecrets = false) { var result = $@" name: move-secrets @@ -106,16 +114,24 @@ private static string GenerateWorkflow(string targetOrg, string targetRepo, stri [System.Reflection.Assembly]::LoadFrom($sodiumPath) $targetPat = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("":$($env:TARGET_PAT)"")) + $sourcePat = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("":$($env:SOURCE_PAT)"")) $publicKeyResponse = Invoke-RestMethod -Uri ""https://api.github.com/repos/$env:TARGET_ORG/$env:TARGET_REPO/actions/secrets/public-key"" -Method ""GET"" -Headers @{{ Authorization = ""Basic $targetPat"" }} $publicKey = [Convert]::FromBase64String($publicKeyResponse.key) $publicKeyId = $publicKeyResponse.key_id - + $secrets = $env:REPO_SECRETS | ConvertFrom-Json + $ignoreSecrets = @(""github_token"", ""SECRETS_MIGRATOR_SOURCE_PAT"", ""SECRETS_MIGRATOR_TARGET_PAT"") + + if ([System.Convert]::ToBoolean($env:IGNORE_ORG_SECRETS)) {{ + $orgSecretsResponse = Invoke-RestMethod -Uri ""https://api.github.com/repos/$env:SOURCE_ORG/$env:SOURCE_REPO/actions/organization-secrets"" -Method ""GET"" -Headers @{{ Authorization = ""Basic $sourcePat"" }} + $ignoreSecrets += $orgSecretsResponse.secrets.name + }} + $secrets | Get-Member -MemberType NoteProperty | ForEach-Object {{ $secretName = $_.Name $secretValue = $secrets.""$secretName"" - if ($secretName -ne ""github_token"" -and $secretName -ne ""SECRETS_MIGRATOR_PAT"") {{ + if ($secretName -notin $ignoreSecrets) {{ Write-Output ""Migrating Secret: $secretName"" $secretBytes = [Text.Encoding]::UTF8.GetBytes($secretValue) $sealedPublicKeyBox = [Sodium.SealedPublicKeyBox]::Create($secretBytes, $publicKey) @@ -135,13 +151,18 @@ private static string GenerateWorkflow(string targetOrg, string targetRepo, stri }} Write-Output ""Cleaning up..."" - Invoke-RestMethod -Uri ""https://api.github.com/repos/${{{{ github.repository }}}}/git/${{{{ github.ref }}}}"" -Method ""DELETE"" -Headers @{{ Authorization = ""Basic $targetPat"" }} - Invoke-RestMethod -Uri ""https://api.github.com/repos/${{{{ github.repository }}}}/actions/secrets/SECRETS_MIGRATOR_PAT"" -Method ""DELETE"" -Headers @{{ Authorization = ""Basic $targetPat"" }} + Invoke-RestMethod -Uri ""https://api.github.com/repos/${{{{ github.repository }}}}/git/${{{{ github.ref }}}}"" -Method ""DELETE"" -Headers @{{ Authorization = ""Basic $sourcePat"" }} + Invoke-RestMethod -Uri ""https://api.github.com/repos/${{{{ github.repository }}}}/actions/secrets/SECRETS_MIGRATOR_TARGET_PAT"" -Method ""DELETE"" -Headers @{{ Authorization = ""Basic $sourcePat"" }} + Invoke-RestMethod -Uri ""https://api.github.com/repos/${{{{ github.repository }}}}/actions/secrets/SECRETS_MIGRATOR_SOURCE_PAT"" -Method ""DELETE"" -Headers @{{ Authorization = ""Basic $sourcePat"" }} env: REPO_SECRETS: ${{{{ toJSON(secrets) }}}} - TARGET_PAT: ${{{{ secrets.SECRETS_MIGRATOR_PAT }}}} + TARGET_PAT: ${{{{ secrets.SECRETS_MIGRATOR_TARGET_PAT }}}} + SOURCE_PAT: ${{{{ secrets.SECRETS_MIGRATOR_SOURCE_PAT }}}} TARGET_ORG: '{targetOrg}' TARGET_REPO: '{targetRepo}' + SOURCE_ORG: '{sourceOrg}' + SOURCE_REPO: '{sourceRepo}' + IGNORE_ORG_SECRETS: '{ignoreOrgSecrets}' shell: pwsh "; diff --git a/src/Properties/launchSettings.json b/src/Properties/launchSettings.json index a336194..d452235 100644 --- a/src/Properties/launchSettings.json +++ b/src/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "SecretsMigrator": { "commandName": "Project", - "commandLineArgs": "migrate-secrets --source-org dylan-smith --source-repo secrets-testing --target-org dylan-smith --target-repo secrets-target --source-pat \"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\" --target-pat \"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\"" + "commandLineArgs": "migrate-secrets --source-org dylan-smith --source-repo secrets-testing --target-org dylan-smith --target-repo secrets-target --source-pat \"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\" --target-pat \"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\" --ignore-org-secrets" } } } \ No newline at end of file diff --git a/src/SecretsMigrator.csproj b/src/SecretsMigrator.csproj index ef4a029..c97ab5c 100644 --- a/src/SecretsMigrator.csproj +++ b/src/SecretsMigrator.csproj @@ -1,12 +1,11 @@ - - + + Exe net6.0 enable disable - @@ -15,5 +14,4 @@ - - + \ No newline at end of file From 14de0474d9b917f275046d8e70050c21f5a7f68f Mon Sep 17 00:00:00 2001 From: "Michelle A. Taal" Date: Wed, 25 Oct 2023 21:40:01 +0200 Subject: [PATCH 2/3] Wait for completion --- src/GithubApi.cs | 30 +++++++++++++++++++++++++++ src/Program.cs | 33 ++++++++++++++++++++++++++++-- src/Properties/launchSettings.json | 2 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/GithubApi.cs b/src/GithubApi.cs index 4dc90ca..432f427 100644 --- a/src/GithubApi.cs +++ b/src/GithubApi.cs @@ -250,5 +250,35 @@ public virtual async Task CreateTreeFromTree(string org, string repo, st return (string)data["sha"]; } + + public virtual async Task GetLatestRunBranchWorkflow(string org, string repo, string branch, string workflowId) + { + var url = $"{_apiUrl}/repos/{org}/{repo}/actions/workflows/{workflowId}/runs?branch={branch}"; + var data = JObject.Parse(@"{""total_count"": 0}"); + + var retryCount = 0; + while (((int)data["total_count"]).Equals(0) && retryCount < 20) + { + await Task.Delay(1000); // start with delay for job to initialize + Console.WriteLine($"Retry: {retryCount}"); + var response = await _client.GetAsync(url); + data = JObject.Parse(response); + retryCount++; + } + + var runId = data["workflow_runs"].OrderByDescending(x => x["run_number"]).First()["id"]; + + return (string)runId; + } + + public virtual async Task<(string status, string conclusion)> GetWorkflowRunStatus(string org, string repo, string runId) + { + var url = $"{_apiUrl}/repos/{org}/{repo}/actions/runs/{runId}"; + + var response = await _client.GetAsync(url); + var data = JObject.Parse(response); + + return ((string)data["status"], (string)data["conclusion"]); + } } } diff --git a/src/Program.cs b/src/Program.cs index 7482f7a..efd405d 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -44,6 +44,10 @@ public static async Task Main(string[] args) { IsRequired = false }; + var waitForCompletion = new Option("--wait-for-completion") + { + IsRequired = false + }; var verbose = new Option("--verbose") { IsRequired = false @@ -56,14 +60,15 @@ public static async Task Main(string[] args) root.AddOption(sourcePat); root.AddOption(targetPat); root.AddOption(ignoreOrgSecrets); + root.AddOption(waitForCompletion); root.AddOption(verbose); - root.Handler = CommandHandler.Create(Invoke); + root.Handler = CommandHandler.Create(Invoke); await root.InvokeAsync(args); } - public static async Task Invoke(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string sourcePat, string targetPat, bool ignoreOrgSecrets = false, bool verbose = false) + public static async Task Invoke(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string sourcePat, string targetPat, bool ignoreOrgSecrets = false, bool waitForCompletion = false, bool verbose = false) { _log.Verbose = verbose; @@ -91,6 +96,30 @@ public static async Task Invoke(string sourceOrg, string sourceRepo, string targ await githubApi.CreateFile(sourceOrg, sourceRepo, branchName, ".github/workflows/migrate-secrets.yml", workflow); _log.LogSuccess($"Secrets migration in progress. Check on status at https://github.com/{sourceOrg}/{sourceRepo}/actions"); + + if (waitForCompletion) + { + var runId = await githubApi.GetLatestRunBranchWorkflow(sourceOrg, sourceRepo, branchName, "migrate-secrets.yml"); + + var status = "queued"; + var conclusion = "neutral"; + while (status != "completed") + { + await Task.Delay(5000); + var result = await githubApi.GetWorkflowRunStatus(sourceOrg, sourceRepo, runId); + status = result.status; + conclusion = result.conclusion; + } + + if (conclusion == "success") + { + _log.LogSuccess("Secrets migration completed successfully."); + } + else + { + _log.LogError("Secrets migration failed."); + } + } } private static string GenerateWorkflow(string sourceOrg, string sourceRepo, string targetOrg, string targetRepo, string branchName, bool ignoreOrgSecrets = false) diff --git a/src/Properties/launchSettings.json b/src/Properties/launchSettings.json index d452235..0f49045 100644 --- a/src/Properties/launchSettings.json +++ b/src/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "SecretsMigrator": { "commandName": "Project", - "commandLineArgs": "migrate-secrets --source-org dylan-smith --source-repo secrets-testing --target-org dylan-smith --target-repo secrets-target --source-pat \"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\" --target-pat \"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\" --ignore-org-secrets" + "commandLineArgs": "migrate-secrets --source-org dylan-smith --source-repo secrets-testing --target-org dylan-smith --target-repo secrets-target --source-pat \"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\" --target-pat \"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\" --ignore-org-secrets --wait-for-completion" } } } \ No newline at end of file From 07e85aaaa64a0ecfc0fe0ab1bff5fc81f4d92026 Mon Sep 17 00:00:00 2001 From: "Michelle A. Taal" Date: Wed, 25 Oct 2023 22:46:14 +0200 Subject: [PATCH 3/3] Clean up --- src/SecretsMigrator.csproj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/SecretsMigrator.csproj b/src/SecretsMigrator.csproj index c97ab5c..4460844 100644 --- a/src/SecretsMigrator.csproj +++ b/src/SecretsMigrator.csproj @@ -1,11 +1,12 @@ - - + + Exe net6.0 enable disable + @@ -14,4 +15,5 @@ + \ No newline at end of file