diff --git a/docs/region-configuration.md b/docs/region-configuration.md index 8343e969..e83391ef 100644 --- a/docs/region-configuration.md +++ b/docs/region-configuration.md @@ -47,26 +47,27 @@ The Onyxia service platform is a Kubernetes cluster but Onyxia is meant to be ex Users can work on Onyxia as a User or as a Group to which they belong. Each user and group can have its own **namespace** which is an isolated space of Kubernetes. -| Key | Default | Description | Example | -|-------------------------------| ------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| `type` | | Type of the platform on which services are launched. Only Kubernetes is supported, Marathon has been removed. | "KUBERNETES" | -| `allowNamespaceCreation` | true | If true, the /onboarding endpoint is enabled and the user will have a namespace created on its first request on a service resource. | true | -| `namespaceLabels` | | Static labels to add to the namespace (at creation and subsequent user logins) | {"zone":"prod"} | -| `namespaceAnnotations` | | Static annotations to add to the namespace (at creation and subsequent user logins) | {"zone":"prod"} | -| `namespaceAnnotationsDynamic` | | Dynamic annotations (currently only based on user JWT token) to add to the namespace (at creation and subsequent user logins). Annotations names will be prefixed with `onyxia_`. `onyxia_last_login_timestamp` is also added. | {"enabled": true, "userAttributes": ["sub", "email"] } | -| `singleNamespace` | true | When true, all users share the same namespace on the service provider. This configuration can be used if a project works on its own Onyxia region. | | -| `userNamespace` | true | When true, all users have a namespace for their work. This configuration can be used if you don't allow a user to have their own space to work and only use project space | | -| `namespacePrefix` | "user-" | User has a personal namespace like namespacePrefix + userId (should only be used when not singleNamespace but not the case) | | -| `groupNamespacePrefix` | "projet-" | User in a group groupId can access the namespace groupeNamespacePrefix + groupId. This prefix is also used for the Vault group directory. | | -| `usernamePrefix` | | If set, the Kubernetes user corresponding to the Onyxia user is named usernamePrefix + userId on impersonation mode, otherwise it is identified only as userId | "user-" | -| `groupPrefix` | | not used | | -| `authenticationMode` | serviceAccount | serviceAccount, impersonate or tokenPassthrough : on serviceAccount mode Onyxia API uses its own serviceAccount (by default admin or cluster-admin), with impersonate mode Onyxia requests the API with user's permissions (helm option `--kube-as-user`). With tokenPassthrough, the authentication token is passed to the API server. | | -| `expose` | | When users request to expose their service, only subdomain of this object domain are allowed | See [Expose properties](#expose-properties) | -| `monitoring` | | Define the URL pattern of the monitoring service that is to be launched with each service. Only for client purposes. | {URLPattern: "https://$NAMESPACE-$INSTANCE.mymonitoring.sspcloud.fr"} | | -| `allowedURIPattern` | "^https://" | Init scripts set by the user have to respect this pattern. | | -| `server` | | Define the configuration of the services provider API server, this value is not served on the API as it contains credentials for the API. | See [Server properties](#server-properties) | -| `k8sPublicEndpoint` | | Define external access to Kubernetes API if available. It helps Onyxia users to directly connect to Kubernetes outside the datalab | See [K8sPublicEndpoint properties](#k8sPublicEndpoint-properties) | -| `quotas` | | Properties setting quotas on how many resources a user can get on the services provider. | See [Quotas properties](#quotas-properties) | +| Key | Default | Description | Example | +|-------------------------------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------| +| `type` | | Type of the platform on which services are launched. Only Kubernetes is supported, Marathon has been removed. | "KUBERNETES" | +| `allowNamespaceCreation` | true | If true, the /onboarding endpoint is enabled and the user will have a namespace created on its first request on a service resource. | true | +| `namespaceLabels` | | Static labels to add to the namespace (at creation and subsequent user logins) | {"zone":"prod"} | +| `namespaceAnnotations` | | Static annotations to add to the namespace (at creation and subsequent user logins) | {"zone":"prod"} | +| `namespaceAnnotationsDynamic` | | Dynamic annotations (currently only based on user JWT token) to add to the namespace (at creation and subsequent user logins). Annotations names will be prefixed with `onyxia_`. `onyxia_last_login_timestamp` is also added. | {"enabled": true, "userAttributes": ["sub", "email"] } | +| `singleNamespace` | true | When true, all users share the same namespace on the service provider. This configuration can be used if a project works on its own Onyxia region. | | +| `userNamespace` | true | When true, all users have a namespace for their work. This configuration can be used if you don't allow a user to have their own space to work and only use project space | | +| `namespacePrefix` | "user-" | User has a personal namespace like namespacePrefix + userId (should only be used when not singleNamespace but not the case) | | +| `groupNamespacePrefix` | "projet-" | User in a group groupId can access the namespace groupeNamespacePrefix + groupId. This prefix is also used for the Vault group directory. | | +| `usernamePrefix` | | If set, the Kubernetes user corresponding to the Onyxia user is named usernamePrefix + userId on impersonation mode, otherwise it is identified only as userId | "user-" | +| `groupPrefix` | | not used | | +| `authenticationMode` | serviceAccount | serviceAccount, impersonate or tokenPassthrough : on serviceAccount mode Onyxia API uses its own serviceAccount (by default admin or cluster-admin), with impersonate mode Onyxia requests the API with user's permissions (helm option `--kube-as-user`). With tokenPassthrough, the authentication token is passed to the API server. | | +| `expose` | | When users request to expose their service, only subdomain of this object domain are allowed | See [Expose properties](#expose-properties) | +| `monitoring` | | Define the URL pattern of the monitoring service that is to be launched with each service. Only for client purposes. | {URLPattern: "https://$NAMESPACE-$INSTANCE.mymonitoring.sspcloud.fr"} | | +| `allowedURIPattern` | "^https://" | Init scripts set by the user have to respect this pattern. | | +| `server` | | Define the configuration of the services provider API server, this value is not served on the API as it contains credentials for the API. | See [Server properties](#server-properties) | +| `k8sPublicEndpoint` | | Define external access to Kubernetes API if available. It helps Onyxia users to directly connect to Kubernetes outside the datalab | See [K8sPublicEndpoint properties](#k8sPublicEndpoint-properties) | +| `quotas` | | Properties setting quotas on how many resources a user can get on the services provider. | See [Quotas properties](#quotas-properties) | +| `helm` | | Properties related to helm flags used by onyxia | See [Quotas properties](#quotas-properties) | @@ -141,6 +142,14 @@ Note : If you want Onyxia to create the ResourceQuota but not override it at eac | `gateways` | [] | List of istio gateways to be used. Should contain at least one element. E.g. `["istio-system/my-gateway"]` | +### Helm properties + +It can be used to add additional flags which will be used when installing, resuming and suspending services running Onyxia. + +| Key | Default | Description | +|------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `forceConflicts` | false | If set server-side apply will force changes against conflicts, also see https://helm.sh/docs/helm/helm_upgrade#options . This might be useful if you have mutating webhooks which take ownership of fields, which would normaly result in Helm 4 to fail (see https://helm.sh/community/hips/hip-0023#conflicts-and-forcing ). | + ## Data properties diff --git a/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmFlags.java b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmFlags.java new file mode 100644 index 00000000..f3e7d54a --- /dev/null +++ b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmFlags.java @@ -0,0 +1,66 @@ +package io.github.inseefrlab.helmwrapper.service; + +public record HelmFlags( + boolean dryRun, + boolean skipTlsVerify, + String timeout, + String caFile, + boolean reuseValues, + boolean forceConflicts, + String serverSide) { + + public static HelmFlags suspendAndResumeFlags( + boolean dryRun, + boolean skipTlsVerify, + String timeout, + String caFile, + boolean forceConflicts, + String serverSide) { + return new HelmFlags( + dryRun, skipTlsVerify, timeout, caFile, true, forceConflicts, serverSide); + } + + public static HelmFlags installFlags( + boolean dryRun, + boolean skipTlsVerify, + String timeout, + String caFile, + boolean forceConflicts, + String serverSide) { + return new HelmFlags( + dryRun, skipTlsVerify, timeout, caFile, false, forceConflicts, serverSide); + } + + /** + * @return cli ready string to use for helm upgrade, ending with a space such that it is safe to + * further append on + */ + public String toHelmUpgradeCliString() { + StringBuilder result = new StringBuilder(); + if (forceConflicts) { + result.append("--force-conflicts "); + } + if (serverSide != null) { + result.append(" --server-side " + serverSide + " "); + } + + if (timeout != null) { + result.append("--timeout " + timeout + " "); + } + + if (skipTlsVerify) { + result.append("--insecure-skip-tls-verify "); + } else if (caFile != null) { + result.append("--ca-file " + System.getenv("CACERTS_DIR") + "/" + caFile + " "); + } + + if (dryRun) { + result.append("--dry-run "); + } + if (reuseValues) { + result.append("--reuse-values "); + } + + return result.toString(); + } +} diff --git a/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java index 664debb5..53b7c1f7 100644 --- a/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java +++ b/helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmInstallService.java @@ -38,111 +38,15 @@ public class HelmInstallService { private static final String MANIFEST_INFO_TYPE = "manifest"; private static final String NOTES_INFO_TYPE = "notes"; - public void resume( - HelmConfiguration configuration, - String chart, - String namespace, - String name, - String version, - boolean dryRun, - final boolean skipTlsVerify, - String timeout, - String caFile) - throws InvalidExitValueException, - IOException, - InterruptedException, - TimeoutException, - IllegalArgumentException { - installChart( - configuration, - chart, - namespace, - name, - version, - dryRun, - null, - Map.of("global.suspend", "false"), - skipTlsVerify, - timeout, - caFile, - true); - } - - public void suspend( - HelmConfiguration configuration, - String chart, - String namespace, - String name, - String version, - boolean dryRun, - final boolean skipTlsVerify, - String timeout, - String caFile) - throws InvalidExitValueException, - IOException, - InterruptedException, - TimeoutException, - IllegalArgumentException { - installChart( - configuration, - chart, - namespace, - name, - version, - dryRun, - null, - Map.of("global.suspend", "true"), - skipTlsVerify, - timeout, - caFile, - true); - } - - public HelmInstaller installChart( - HelmConfiguration configuration, - String chart, - String namespace, - String name, - String version, - boolean dryRun, - File values, - Map env, - final boolean skipTlsVerify, - String timeout, - String caFile) - throws InvalidExitValueException, - IOException, - InterruptedException, - TimeoutException, - IllegalArgumentException { - return installChart( - configuration, - chart, - namespace, - name, - version, - dryRun, - values, - env, - skipTlsVerify, - timeout, - caFile, - false); - } - public HelmInstaller installChart( HelmConfiguration configuration, String chart, String namespace, String name, String version, - boolean dryRun, File values, Map env, - final boolean skipTlsVerify, - String timeout, - String caFile, - boolean reuseValues) + HelmFlags additionalFlags) throws InvalidExitValueException, IOException, InterruptedException, @@ -150,15 +54,7 @@ public HelmInstaller installChart( IllegalArgumentException { StringBuilder command = new StringBuilder("helm upgrade --install --history-max 0 "); - if (timeout != null) { - command.append("--timeout " + timeout + " "); - } - - if (skipTlsVerify) { - command.append("--insecure-skip-tls-verify "); - } else if (caFile != null) { - command.append("--ca-file " + System.getenv("CACERTS_DIR") + "/" + caFile + " "); - } + command.append(additionalFlags.toHelmUpgradeCliString()); if (name != null) { if (!helmNamePattern.matcher(name).matches() || name.length() > 53) { @@ -196,12 +92,6 @@ public HelmInstaller installChart( if (env != null) { command.append(buildEnvVar(env)); } - if (dryRun) { - command.append(" --dry-run"); - } - if (reuseValues) { - command.append(" --reuse-values"); - } String res = Command.executeAndGetResponseAsJson(configuration, command.toString()) .getOutput() diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java index 657e94cf..0e0bc735 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java @@ -23,6 +23,7 @@ import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.WatcherException; +import io.github.inseefrlab.helmwrapper.service.HelmFlags; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; @@ -297,6 +298,14 @@ private void suspendOrResume(Region region, Project project, String serviceId, b throw new IllegalStateException("Catalog " + catalogId + " is not available anymore"); } + var flags = + HelmFlags.suspendAndResumeFlags( + false, + catalog.get().getSkipTlsVerify(), + catalog.get().getTimeout(), + catalog.get().getCaFile(), + region.getServices().getHelm().getForceConflicts(), + region.getServices().getHelm().getServerSide()); if (suspend) { helmAppsService.suspend( region, @@ -306,10 +315,7 @@ private void suspendOrResume(Region region, Project project, String serviceId, b version, user, serviceId, - catalog.get().getSkipTlsVerify(), - catalog.get().getTimeout(), - catalog.get().getCaFile(), - false); + flags); } else { helmAppsService.resume( region, @@ -319,10 +325,7 @@ private void suspendOrResume(Region region, Project project, String serviceId, b version, user, serviceId, - catalog.get().getSkipTlsVerify(), - catalog.get().getTimeout(), - catalog.get().getCaFile(), - false); + flags); } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/AppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/AppsService.java index 5927c96b..f3e28655 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/AppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/AppsService.java @@ -11,6 +11,7 @@ import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; +import io.github.inseefrlab.helmwrapper.service.HelmFlags; import io.github.inseefrlab.helmwrapper.service.HelmInstallService; import java.io.IOException; import java.text.ParseException; @@ -69,10 +70,7 @@ void resume( String version, User user, String serviceId, - boolean skipTlsVerify, - String timeout, - String caFile, - boolean dryRun) + HelmFlags flags) throws IOException, InterruptedException, TimeoutException; void suspend( @@ -83,9 +81,6 @@ void suspend( String version, User user, String serviceId, - boolean skipTlsVerify, - String timeout, - String caFile, - boolean dryRun) + HelmFlags flags) throws IOException, InterruptedException, TimeoutException; } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index 35319101..ae010ece 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -30,6 +30,7 @@ import io.github.inseefrlab.helmwrapper.model.HelmInstaller; import io.github.inseefrlab.helmwrapper.model.HelmLs; import io.github.inseefrlab.helmwrapper.model.HelmReleaseInfo; +import io.github.inseefrlab.helmwrapper.service.HelmFlags; import io.github.inseefrlab.helmwrapper.service.HelmInstallService; import io.github.inseefrlab.helmwrapper.service.HelmInstallService.MultipleServiceFound; import java.io.File; @@ -120,12 +121,15 @@ public Collection installApp( namespaceId, requestDTO.getName(), pkg.getVersion(), - requestDTO.isDryRun(), values, null, - skipTlsVerify, - timeout, - caFile); + HelmFlags.installFlags( + requestDTO.isDryRun(), + skipTlsVerify, + timeout, + caFile, + region.getServices().getHelm().getForceConflicts(), + region.getServices().getHelm().getServerSide())); InstallServiceEvent installServiceEvent = new InstallServiceEvent( user.getIdep(), @@ -410,24 +414,10 @@ public void suspend( String version, User user, String serviceId, - boolean skipTlsVerify, - String timeout, - String caFile, - boolean dryRun) + HelmFlags flags) throws IOException, InterruptedException, TimeoutException { suspendOrResume( - region, - project, - catalogId, - chartName, - version, - user, - serviceId, - skipTlsVerify, - timeout, - caFile, - dryRun, - true); + region, project, catalogId, chartName, version, user, serviceId, flags, true); } @Override @@ -439,24 +429,10 @@ public void resume( String version, User user, String serviceId, - boolean skipTlsVerify, - String timeout, - String caFile, - boolean dryRun) + HelmFlags flags) throws IOException, InterruptedException, TimeoutException { suspendOrResume( - region, - project, - catalogId, - chartName, - version, - user, - serviceId, - skipTlsVerify, - timeout, - caFile, - dryRun, - false); + region, project, catalogId, chartName, version, user, serviceId, flags, false); } public void suspendOrResume( @@ -467,39 +443,22 @@ public void suspendOrResume( String version, User user, String serviceId, - boolean skipTlsVerify, - String timeout, - String caFile, - boolean dryRun, + HelmFlags flags, boolean suspend) throws IOException, InterruptedException, TimeoutException { String namespaceId = kubernetesService.determineNamespaceAndCreateIfNeeded(region, project, user); - if (suspend) { - getHelmInstallService() - .suspend( - getHelmConfiguration(region, user), - catalogId + "/" + chartName, - namespaceId, - serviceId, - version, - dryRun, - skipTlsVerify, - timeout, - caFile); - } else { - getHelmInstallService() - .resume( - getHelmConfiguration(region, user), - catalogId + "/" + chartName, - namespaceId, - serviceId, - version, - dryRun, - skipTlsVerify, - timeout, - caFile); - } + var suspendEnv = Map.of(SUSPEND_KEY, suspend ? "true" : "false"); + getHelmInstallService() + .installChart( + getHelmConfiguration(region, user), + catalogId + "/" + chartName, + namespaceId, + serviceId, + version, + null, + suspendEnv, + flags); SuspendResumeServiceEvent event = new SuspendResumeServiceEvent( user.getIdep(), namespaceId, serviceId, chartName, catalogId, suspend); diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java index 8fb6f6ed..5cb8601c 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java @@ -178,6 +178,7 @@ public static class Services { private Map namespaceLabels = new HashMap<>(); private Map namespaceAnnotations = new HashMap<>(); private boolean userNamespace = true; + private String namespacePrefix = "user-"; private String groupNamespacePrefix = "projet-"; private String usernamePrefix; @@ -188,6 +189,7 @@ public static class Services { private Monitoring monitoring; private String allowedURIPattern = "^https://"; private Quotas quotas = new Quotas(); + private Helm helm = new Helm(); /*** * @Deprecated since v3 @@ -346,6 +348,14 @@ public void setQuotas(Quotas quotas) { this.quotas = quotas; } + public Helm getHelm() { + return helm; + } + + public void setHelm(Helm helm) { + this.helm = helm; + } + public static enum AuthenticationMode { @JsonProperty("impersonate") IMPERSONATE, @@ -433,6 +443,30 @@ public Map getRolesQuota() { return rolesQuota; } } + + public static class Helm { + + /** If set server-side apply will force changes against conflicts */ + private boolean forceConflicts = false; + + private String serverSide = "auto"; + + public boolean getForceConflicts() { + return forceConflicts; + } + + public void setForceConflicts(boolean forceConflicts) { + this.forceConflicts = forceConflicts; + } + + public String getServerSide() { + return serverSide; + } + + public void setServerSide(String serverSide) { + this.serverSide = serverSide; + } + } } public static class Monitoring {