feat(container): support multiple registry credentials

GZTimeWalker · GZTimeWalker · commit a51c3da17da6 · 2025-03-09T12:36:49.000+08:00
diff --git a/src/GZCTF/Extensions/Startup/ServicesExtension.cs b/src/GZCTF/Extensions/Startup/ServicesExtension.cs
@@ -23,13 +23,23 @@ static void AddConfig<TConfig>(this WebApplicationBuilder builder)
     internal static void AddServiceConfigurations(this WebApplicationBuilder builder)
     {
         builder.AddConfig<EmailConfig>();
-        builder.AddConfig<RegistryConfig>();
         builder.AddConfig<AccountPolicy>();
         builder.AddConfig<GlobalConfig>();
         builder.AddConfig<ContainerPolicy>();
         builder.AddConfig<ContainerProvider>();
 
-        var forwardedOptions =
+        builder.Services.Configure<RegistrySet<RegistryConfig>>(builder.Configuration.GetSection("Registries"));
+
+        RegistryConfig? oldConfig = builder.Configuration.GetSection(nameof(RegistryConfig)).Get<RegistryConfig>();
+        if (!string.IsNullOrWhiteSpace(oldConfig?.ServerAddress))
+            // Add old config to new config set
+            builder.Services.Configure<RegistrySet<RegistryConfig>>(set =>
+            {
+                if (!set.TryAdd(oldConfig.ServerAddress, oldConfig))
+                    set[oldConfig.ServerAddress] = oldConfig;
+            });
+
+        ForwardedOptions? forwardedOptions =
             builder.Configuration.GetSection(nameof(ForwardedOptions)).Get<ForwardedOptions>();
         if (forwardedOptions is null)
             builder.Services.Configure<ForwardedHeadersOptions>(options =>
diff --git a/src/GZCTF/Models/Internal/Configs.cs b/src/GZCTF/Models/Internal/Configs.cs
@@ -293,11 +293,30 @@ public class KubernetesConfig
     public string[]? Dns { get; set; }
 }
 
+public class RegistrySet<T> : Dictionary<string, T>
+where T : class
+{
+    public T? GetForImage(string image)
+    {
+        if (!Uri.TryCreate(image, UriKind.Absolute, out var uri))
+            return null;
+
+        var host = uri.Host;
+        if (!uri.IsDefaultPort)
+            host = $"{host}:{uri.Port}";
+
+        return TryGetValue(host, out var config) ? config : null;
+    }
+}
+
 public class RegistryConfig
 {
     public string? ServerAddress { get; set; }
     public string? UserName { get; set; }
     public string? Password { get; set; }
+
+    public bool Valid => !string.IsNullOrEmpty(UserName) &&
+                       !string.IsNullOrEmpty(Password);
 }
 
 #endregion
diff --git a/src/GZCTF/Models/Internal/ContainerInfo.cs b/src/GZCTF/Models/Internal/ContainerInfo.cs
diff --git a/src/GZCTF/Repositories/ParticipationRepository.cs b/src/GZCTF/Repositories/ParticipationRepository.cs
@@ -101,7 +101,7 @@ public async Task UpdateParticipation(Participation part, ParticipationEditModel
         }
 
         await trans.CommitAsync(token);
-        
+
         if (needFlush)
             await cacheHelper.FlushScoreboardCache(part.GameId, token);
     }
diff --git a/src/GZCTF/Services/Container/Manager/DockerManager.cs b/src/GZCTF/Services/Container/Manager/DockerManager.cs
@@ -121,8 +121,10 @@ await _client.Containers.RemoveContainerAsync(container.ContainerId,
                 StaticLocalizer[nameof(Resources.Program.ContainerManager_PullContainerImage), config.Image],
                 TaskStatus.Pending, LogLevel.Information);
 
+            var auth = _meta.AuthConfigs.GetForImage(config.Image);
+
             // pull the image and retry
-            await _client.Images.CreateImageAsync(new() { FromImage = config.Image }, _meta.Auth,
+            await _client.Images.CreateImageAsync(new() { FromImage = config.Image }, auth,
                 new Progress<JSONMessage>(msg =>
                 {
                     Console.WriteLine($@"{msg.Status}|{msg.ProgressMessage}|{msg.ErrorMessage}");
diff --git a/src/GZCTF/Services/Container/Manager/KubernetesManager.cs b/src/GZCTF/Services/Container/Manager/KubernetesManager.cs
@@ -43,14 +43,14 @@ public KubernetesManager(IContainerProvider<Kubernetes, KubernetesMetadata> prov
             return null;
         }
 
-        var authSecretName = _meta.AuthSecretName;
-        var options = _meta.Config;
+        var authSecretName = _meta.AuthSecretNames.GetForImage(config.Image);
+        KubernetesConfig options = _meta.Config;
 
         var chalImage = imageName.ToValidRFC1123String("chal");
 
         var name = $"{chalImage}-{Guid.NewGuid().ToString("N")[..16]}";
 
-        var pod = new V1Pod("v1", "Pod")
+        var pod = new V1Pod
         {
             Metadata = new V1ObjectMeta
             {
@@ -67,10 +67,6 @@ public KubernetesManager(IContainerProvider<Kubernetes, KubernetesMetadata> prov
             },
             Spec = new V1PodSpec
             {
-                ImagePullSecrets =
-                    authSecretName is null
-                        ? Array.Empty<V1LocalObjectReference>()
-                        : new List<V1LocalObjectReference> { new() { Name = authSecretName } },
                 DnsPolicy = "None",
                 DnsConfig = new() { Nameservers = options.Dns ?? ["223.5.5.5", "114.114.114.114"] },
                 EnableServiceLinks = false,
@@ -113,6 +109,9 @@ config.Flag is null
             }
         };
 
+        if (authSecretName is not null)
+            pod.Spec.ImagePullSecrets.Add(new() { Name = authSecretName });
+
         try
         {
             pod = await _client.CreateNamespacedPodAsync(pod, options.Namespace, cancellationToken: token);
diff --git a/src/GZCTF/Services/Container/Manager/SwarmManager.cs b/src/GZCTF/Services/Container/Manager/SwarmManager.cs
@@ -81,7 +81,7 @@ public async Task DestroyContainerAsync(Models.Data.Container container, Cancell
             return null;
         }
 
-        var parameters = GetServiceCreateParameters(config);
+        ServiceCreateParameters parameters = GetServiceCreateParameters(config);
         ServiceCreateResponse? serviceRes;
         var retry = 0;
 
@@ -171,7 +171,7 @@ public async Task DestroyContainerAsync(Models.Data.Container container, Cancell
     ServiceCreateParameters GetServiceCreateParameters(ContainerConfig config) =>
         new()
         {
-            RegistryAuth = _meta.Auth,
+            RegistryAuth = _meta.AuthConfigs.GetForImage(config.Image),
             Service = new()
             {
                 Name = DockerMetadata.GetName(config),
diff --git a/src/GZCTF/Services/Container/Provider/DockerProvider.cs b/src/GZCTF/Services/Container/Provider/DockerProvider.cs
@@ -16,7 +16,7 @@ public class DockerMetadata : ContainerProviderMetadata
     /// <summary>
     /// Docker 鉴权用配置
     /// </summary>
-    public AuthConfig? Auth { get; set; }
+    public RegistrySet<AuthConfig> AuthConfigs { get; set; } = new();
 
     /// <summary>
     /// 根据配置获取容器名称
@@ -33,7 +33,7 @@ public class DockerProvider : IContainerProvider<DockerClient, DockerMetadata>
     readonly DockerClient _dockerClient;
     readonly DockerMetadata _dockerMeta;
 
-    public DockerProvider(IOptions<ContainerProvider> options, IOptions<RegistryConfig> registry,
+    public DockerProvider(IOptions<ContainerProvider> options, IOptions<RegistrySet<RegistryConfig>> registriesOptions,
         ILogger<DockerProvider> logger)
     {
         _dockerMeta = new()
@@ -54,10 +54,16 @@ public DockerProvider(IOptions<ContainerProvider> options, IOptions<RegistryConf
 
         _dockerClient = cfg.CreateClient();
 
-        // Auth for registry
-        if (!string.IsNullOrWhiteSpace(registry.Value.UserName) && !string.IsNullOrWhiteSpace(registry.Value.Password))
-            _dockerMeta.Auth =
-                new AuthConfig { Username = registry.Value.UserName, Password = registry.Value.Password };
+        var registries = registriesOptions.Value;
+
+        foreach (var registry in registries.Where(registry =>
+                     registry.Value.Valid))
+        {
+            var authConfig = new AuthConfig { Username = registry.Value.UserName, Password = registry.Value.Password };
+
+            if (!_dockerMeta.AuthConfigs.TryAdd(registry.Key, authConfig))
+                _dockerMeta.AuthConfigs[registry.Key] = authConfig;
+        }
 
         logger.SystemLog(
             StaticLocalizer[nameof(Resources.Program.ContainerProvider_DockerInited),
diff --git a/src/GZCTF/Services/Container/Provider/KubernetesProvider.cs b/src/GZCTF/Services/Container/Provider/KubernetesProvider.cs
@@ -11,7 +11,7 @@ public class KubernetesMetadata : ContainerProviderMetadata
     /// <summary>
     /// 容器注册表鉴权 Secret 名称
     /// </summary>
-    public string? AuthSecretName { get; set; }
+    public RegistrySet<string> AuthSecretNames { get; set; } = new();
 
     /// <summary>
     /// K8s 集群 Host IP
@@ -31,7 +31,7 @@ public class KubernetesProvider : IContainerProvider<Kubernetes, KubernetesMetad
     readonly Kubernetes _kubernetesClient;
     readonly KubernetesMetadata _kubernetesMetadata;
 
-    public KubernetesProvider(IOptions<RegistryConfig> registry, IOptions<ContainerProvider> options,
+    public KubernetesProvider(IOptions<RegistrySet<RegistryConfig>> registries, IOptions<ContainerProvider> options,
         ILogger<KubernetesProvider> logger)
     {
         _kubernetesMetadata = new()
@@ -63,21 +63,9 @@ public KubernetesProvider(IOptions<RegistryConfig> registry, IOptions<ContainerP
 
         _kubernetesClient = new Kubernetes(config);
 
-        var registryValue = registry.Value;
-        var withAuth = !string.IsNullOrWhiteSpace(registryValue.ServerAddress)
-                       && !string.IsNullOrWhiteSpace(registryValue.UserName)
-                       && !string.IsNullOrWhiteSpace(registryValue.Password);
-
-        if (withAuth)
-        {
-            var padding =
-                $"{registryValue.UserName}@{registryValue.Password}@{registryValue.ServerAddress}".ToMD5String();
-            _kubernetesMetadata.AuthSecretName = $"{registryValue.UserName}-{padding}".ToValidRFC1123String("secret");
-        }
-
         try
         {
-            InitKubernetes(withAuth, registryValue);
+            InitKubernetes(registries.Value);
         }
         catch (Exception e)
         {
@@ -96,7 +84,7 @@ public KubernetesProvider(IOptions<RegistryConfig> registry, IOptions<ContainerP
 
     public KubernetesMetadata GetMetadata() => _kubernetesMetadata;
 
-    void InitKubernetes(bool withAuth, RegistryConfig? registry)
+    void InitKubernetes(RegistrySet<RegistryConfig> registries)
     {
         if (_kubernetesClient.CoreV1.ListNamespace().Items
             .All(ns => ns.Metadata.Name != _kubernetesMetadata.Config.Namespace))
@@ -133,43 +121,46 @@ void InitKubernetes(bool withAuth, RegistryConfig? registry)
                 }
             }, _kubernetesMetadata.Config.Namespace);
 
-        if (!withAuth || registry?.ServerAddress is null)
-            return;
+        // create auth secrets for registries
+        foreach (KeyValuePair<string, RegistryConfig> registry in registries.Where(registry => registry.Value.Valid))
+            InsertRegistrySecret(registry.Key, registry.Value);
+    }
+
+    void InsertRegistrySecret(string address, RegistryConfig registry)
+    {
+        var padding = $"GZCTF@{registry.UserName}@{address}".ToMD5String();
+        var secretName = $"{registry.UserName}-{padding}".ToValidRFC1123String("secret");
 
         var auth = Codec.Base64.Encode($"{registry.UserName}:{registry.Password}");
         var dockerJsonObj = new DockerRegistryOptions(
-            new Dictionary<string, DockerRegistryEntry>
-            {
-                [registry.ServerAddress] = new(auth, registry.UserName, registry.Password)
-            }
+            new Dictionary<string, DockerRegistryEntry> { [address] = new(auth, registry.UserName, registry.Password) }
         );
 
         var dockerJsonBytes =
             JsonSerializer.SerializeToUtf8Bytes(dockerJsonObj, AppJsonSerializerContext.Default.DockerRegistryOptions);
         var secret = new V1Secret
         {
             Metadata =
-                new V1ObjectMeta
-                {
-                    Name = _kubernetesMetadata.AuthSecretName,
-                    NamespaceProperty = _kubernetesMetadata.Config.Namespace
-                },
+                new V1ObjectMeta { Name = secretName, NamespaceProperty = _kubernetesMetadata.Config.Namespace },
             Data = new Dictionary<string, byte[]> { [".dockerconfigjson"] = dockerJsonBytes },
             Type = "kubernetes.io/dockerconfigjson"
         };
 
         try
         {
-            _kubernetesClient.CoreV1.ReplaceNamespacedSecret(secret, _kubernetesMetadata.AuthSecretName,
+            _kubernetesClient.CoreV1.ReplaceNamespacedSecret(secret, secretName,
                 _kubernetesMetadata.Config.Namespace);
         }
         catch
         {
             _kubernetesClient.CoreV1.CreateNamespacedSecret(secret, _kubernetesMetadata.Config.Namespace);
         }
+
+        if (!_kubernetesMetadata.AuthSecretNames.TryAdd(address, secretName))
+            _kubernetesMetadata.AuthSecretNames[address] = secretName;
     }
 }
 
-internal record DockerRegistryOptions(Dictionary<string, DockerRegistryEntry> auths);
+internal record DockerRegistryOptions(Dictionary<string, DockerRegistryEntry> Auths);
 
-internal record DockerRegistryEntry(string auth, string? username, string? password);
+internal record DockerRegistryEntry(string Auth, string? Username, string? Password);

Original file line number	Diff line number	Diff line change
`@@ -101,7 +101,7 @@ public async Task UpdateParticipation(Participation part, ParticipationEditModel`
`101`	`101`	`}`
`102`	`102`
`103`	`103`	`await trans.CommitAsync(token);`
`104`		`-`
	`104`	`+`
`105`	`105`	`if (needFlush)`
`106`	`106`	`await cacheHelper.FlushScoreboardCache(part.GameId, token);`
`107`	`107`	`}`
Original file line number	Diff line number	Diff line change
`@@ -43,14 +43,14 @@ public KubernetesManager(IContainerProvider<Kubernetes, KubernetesMetadata> prov`
`43`	`43`	`return null;`
`44`	`44`	`}`
`45`	`45`
`46`		`- var authSecretName = _meta.AuthSecretName;`
`47`		`- var options = _meta.Config;`
	`46`	`+ var authSecretName = _meta.AuthSecretNames.GetForImage(config.Image);`
	`47`	`+ KubernetesConfig options = _meta.Config;`
`48`	`48`
`49`	`49`	`var chalImage = imageName.ToValidRFC1123String("chal");`
`50`	`50`
`51`	`51`	`var name = $"{chalImage}-{Guid.NewGuid().ToString("N")[..16]}";`
`52`	`52`
`53`		`- var pod = new V1Pod("v1", "Pod")`
	`53`	`+ var pod = new V1Pod`
`54`	`54`	`{`
`55`	`55`	`Metadata = new V1ObjectMeta`
`56`	`56`	`{`
`@@ -67,10 +67,6 @@ public KubernetesManager(IContainerProvider<Kubernetes, KubernetesMetadata> prov`
`67`	`67`	`},`
`68`	`68`	`Spec = new V1PodSpec`
`69`	`69`	`{`
`70`		`- ImagePullSecrets =`
`71`		`- authSecretName is null`
`72`		`- ? Array.Empty<V1LocalObjectReference>()`
`73`		`- : new List<V1LocalObjectReference> { new() { Name = authSecretName } },`
`74`	`70`	`DnsPolicy = "None",`
`75`	`71`	`DnsConfig = new() { Nameservers = options.Dns ?? ["223.5.5.5", "114.114.114.114"] },`
`76`	`72`	`EnableServiceLinks = false,`
`@@ -113,6 +109,9 @@ config.Flag is null`
`113`	`109`	`}`
`114`	`110`	`};`
`115`	`111`
	`112`	`+ if (authSecretName is not null)`
	`113`	`+ pod.Spec.ImagePullSecrets.Add(new() { Name = authSecretName });`
	`114`	`+`
`116`	`115`	`try`
`117`	`116`	`{`
`118`	`117`	`pod = await _client.CreateNamespacedPodAsync(pod, options.Namespace, cancellationToken: token);`
Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@ public async Task DestroyContainerAsync(Models.Data.Container container, Cancell`
`81`	`81`	`return null;`
`82`	`82`	`}`
`83`	`83`
`84`		`- var parameters = GetServiceCreateParameters(config);`
	`84`	`+ ServiceCreateParameters parameters = GetServiceCreateParameters(config);`
`85`	`85`	`ServiceCreateResponse? serviceRes;`
`86`	`86`	`var retry = 0;`
`87`	`87`
`@@ -171,7 +171,7 @@ public async Task DestroyContainerAsync(Models.Data.Container container, Cancell`
`171`	`171`	`ServiceCreateParameters GetServiceCreateParameters(ContainerConfig config) =>`
`172`	`172`	`new()`
`173`	`173`	`{`
`174`		`- RegistryAuth = _meta.Auth,`
	`174`	`+ RegistryAuth = _meta.AuthConfigs.GetForImage(config.Image),`
`175`	`175`	`Service = new()`
`176`	`176`	`{`
`177`	`177`	`Name = DockerMetadata.GetName(config),`