[ENH] Include Backup-Restore service components

src/_nebari/deploy.py

@@ -15,6 +15,7 @@ def deploy_configuration(
     stages: List[hookspecs.NebariStage],
     disable_prompt: bool = False,
     disable_checks: bool = False,
+    stage_force_unlock: str = None,
 ) -> Dict[str, Any]:
     if config.prevent_deploy:
         raise ValueError(
@@ -47,12 +48,26 @@ def deploy_configuration(
 
     with timer(logger, "deploying Nebari"):
         stage_outputs = {}
+        force_unlock = False
         with contextlib.ExitStack() as stack:
             for stage in stages:
                 s: hookspecs.NebariStage = stage(
                     output_directory=pathlib.Path.cwd(), config=config
                 )
-                stack.enter_context(s.deploy(stage_outputs, disable_prompt))
+                print(f"Deploying stage {s.name}")
+                print("stage_force_unlock", stage_force_unlock)
+                if stage_force_unlock:
+                    _unlock_stage_name, _unlock_state_id = stage_force_unlock.split(":")
+                    if _unlock_stage_name == s.name:
+                        force_unlock = _unlock_state_id
+                    print(f"Force unlocking stage {s.name} :: {force_unlock}")
+                stack.enter_context(
+                    s.deploy(
+                        stage_outputs=stage_outputs,
+                        disable_prompt=disable_prompt,
+                        force_unlock=force_unlock,
+                    )
+                )
 
                 if not disable_checks:
                     s.check(stage_outputs, disable_prompt)

src/_nebari/provider/terraform.py

@@ -28,6 +28,7 @@ def deploy(
     terraform_import: bool = False,
     terraform_apply: bool = True,
     terraform_destroy: bool = False,
+    terraform_lock_id: str = None,
     input_vars: Dict[str, Any] = {},
     state_imports: List[Any] = [],
 ):
@@ -46,6 +47,10 @@ def deploy(
       terraform_destroy: whether to run `terraform destroy` default
         False
 
+      terraform_lock_id: Used to toggle the force-unlock feature of a given stage based
+      on the provided lock_id.
+        None
+
       input_vars: supply values for "variable" resources within
         terraform module
 
@@ -61,6 +66,9 @@ def deploy(
         if terraform_init:
             init(directory)
 
+        if terraform_lock_id:
+            force_unlock(directory, terraform_lock_id)
+
         if terraform_import:
             for addr, id in state_imports:
                 tfimport(
@@ -139,6 +147,14 @@ def init(directory=None, upgrade=True):
         run_terraform_subprocess(command, cwd=directory, prefix="terraform")
 
 
+def force_unlock(directory=None, lock_id=None):
+    logger.info(f"terraform unlock directory={directory}")
+    with timer(logger, "terraform unlock"):
+        run_terraform_subprocess(
+            ["force-unlock", "-force", lock_id], cwd=directory, prefix="terraform"
+        )
+
+
 def apply(directory=None, targets=None, var_files=None):
     targets = targets or []
     var_files = var_files or []

src/_nebari/stages/base.py

@@ -284,13 +284,19 @@ def deploy(
         stage_outputs: Dict[str, Dict[str, Any]],
         disable_prompt: bool = False,
         terraform_init: bool = True,
+        force_unlock: bool = False,
     ):
         deploy_config = dict(
             directory=str(self.output_directory / self.stage_prefix),
             input_vars=self.input_vars(stage_outputs),
             terraform_init=terraform_init,
         )
         state_imports = self.state_imports()
+        print("Deploying terraform resources for", self.name)
+        print("Force unlock:", force_unlock)
+        if force_unlock:
+            deploy_config["terraform_lock_id"] = force_unlock
+
         if state_imports:
             deploy_config["terraform_import"] = True
             deploy_config["state_imports"] = state_imports

src/_nebari/stages/infrastructure/__init__.py

@@ -963,9 +963,12 @@ def post_deploy(
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: bool = False,
     ):
-        with super().deploy(stage_outputs, disable_prompt):
+        with super().deploy(stage_outputs, disable_prompt, force_unlock=force_unlock):
             with kubernetes_provider_context(
                 stage_outputs["stages/" + self.name]["kubernetes_credentials"]["value"]
             ):

src/_nebari/stages/kubernetes_keycloak/__init__.py

@@ -297,9 +297,12 @@ def _attempt_keycloak_connection(
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: bool = False,
     ):
-        with super().deploy(stage_outputs, disable_prompt):
+        with super().deploy(stage_outputs, disable_prompt, force_unlock=force_unlock):
             with keycloak_provider_context(
                 stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"]
             ):

src/_nebari/stages/kubernetes_services/__init__.py

@@ -190,6 +190,18 @@ def check_default(cls, value):
         return value
 
 
+class BackupRestoreStorage(schema.Base):
+    type: str
+    config: Dict[str, Any] = {}
+
+
+class BackupRestore(schema.Base):
+    enabled: bool = False
+    storage: BackupRestoreStorage = BackupRestoreStorage(type="s3")
+    image: str = "nebari/nebari-backup-restore"
+    image_tag: str = "main"
+
+
 class CondaEnvironment(schema.Base):
     name: str
     channels: Optional[List[str]] = None
@@ -368,6 +380,7 @@ class InputSchema(schema.Base):
     jupyterlab: JupyterLab = JupyterLab()
     jhub_apps: JHubApps = JHubApps()
     ceph: RookCeph = RookCeph()
+    backup_restore: BackupRestore = BackupRestore()
 
     def _set_storage_type_default_value(self):
         if self.storage.type is None:
@@ -519,6 +532,13 @@ class ArgoWorkflowsInputVars(schema.Base):
     )
 
 
+class BackupRestoreInputVars(schema.Base):
+    backup_restore_enabled: bool = Field(alias="backup-restore-enabled")
+    backup_restore_storage: BackupRestoreStorage = Field(alias="backup-restore-storage")
+    backup_restore_image: str = Field(alias="backup-restore-image")
+    backup_restore_image_tag: str = Field(alias="backup-restore-image-tag")
+
+
 class KubernetesServicesStage(NebariTerraformStage):
     name = "07-kubernetes-services"
     priority = 70
@@ -687,6 +707,14 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             keycloak_read_only_user_credentials=keycloak_read_only_user_credentials,
         )
 
+        backup_restore_vars = BackupRestoreInputVars(
+            backup_restore_enabled=self.config.backup_restore.enabled,
+            backup_restore_storage=self.config.backup_restore.storage,
+            backup_restore_services=self.config.backup_restore.services,
+            backup_restore_image=self.config.backup_restore.image,
+            backup_restore_image_tag=self.config.backup_restore.image_tag,
+        )
+
         return {
             **kubernetes_services_vars.model_dump(by_alias=True),
             **rook_ceph_vars.model_dump(by_alias=True),
@@ -696,6 +724,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             **monitoring_vars.model_dump(by_alias=True),
             **argo_workflows_vars.model_dump(by_alias=True),
             **telemetry_vars.model_dump(by_alias=True),
+            **backup_restore_vars.model_dump(by_alias=True),
         }
 
     def check(

src/_nebari/stages/kubernetes_services/template/backup-restore.tf

@@ -0,0 +1,29 @@
+variable "backup-restore-image" {
+  description = "The image to use for the backup-restore service"
+  default     = "vcerutti/nebari-backup-restore"
+}
+
+variable "backup-restore-image-tag" {
+  description = "The tag of the image to use for the backup-restore service"
+  default     = "latest"
+}
+
+variable "backup-restore-clients" {
+  description = "List of clients that can access the backup-restore server by API"
+  type        = list(string)
+  default     = ["nebari-cli"]
+}
+
+module "kubernetes-backup-restore-server" {
+  source = "./modules/kubernetes/services/backup-restore"
+
+  name      = "nebari"
+  namespace = var.environment
+
+  external-url = var.endpoint
+  realm_id     = var.realm_id
+
+  backup-restore-image     = var.backup-restore-image
+  backup-restore-image-tag = var.backup-restore-image-tag
+  clients                  = var.backup-restore-clients
+}

src/_nebari/stages/kubernetes_services/template/backup_restore.tf

@@ -0,0 +1,34 @@
+variable "backup-restore-enabled" {
+  description = "Enable backup-restore service"
+  type        = bool
+  default     = false
+}
+
+variable "backup-restore-storage" {
+  description = "Storage backend for backup-restore"
+  type        = map(string)
+  default     = {}
+}
+
+variable "backup-restore-image" {
+  description = "The image to use for the backup-restore service"
+  type        = string
+}
+
+variable "backup-restore-image-tag" {
+  description = "The tag of the image to use for the backup-restore service"
+  type        = string
+}
+
+module "nebari-backup-restore" {
+  count  = var.backup-restore-enabled ? 1 : 0
+  source = "./modules/kubernetes/services/backup-restore"
+
+  external-url = var.endpoint
+  realm_id     = "nebari"
+  image        = var.backup-restore-image
+  storage      = var.backup-restore-storage
+  image_tag    = var.backup-restore-image-tag
+  namespace    = var.environment
+  clients      = ["nebari-cli"]
+}

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/backup-restore/ingress.tf

@@ -0,0 +1,26 @@
+resource "kubernetes_manifest" "backup-restore-api" {
+  manifest = {
+    apiVersion = "traefik.containo.us/v1alpha1"
+    kind       = "IngressRoute"
+    metadata = {
+      name      = "backup-restore-api"
+      namespace = var.namespace
+    }
+    spec = {
+      entryPoints = ["backup-restore"]
+      routes = [
+        {
+          kind  = "Rule"
+          match = "Host(`${var.external-url}`)"
+          services = [
+            {
+              name      = kubernetes_deployment.backup_restore.metadata.0.name
+              port      = 9000
+              namespace = var.namespace
+            }
+          ]
+        }
+      ]
+    }
+  }
+}