IDP Arc portal integration with OSB

applications/accounts-api/Dockerfile

@@ -13,5 +13,3 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 COPY backend /usr/src/app
 
 RUN pip3 install -e .
-
-ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app

applications/accounts-api/backend/accounts_api/services/user_service.py

@@ -61,7 +61,15 @@ def get_users(query: str) -> typing.List[User]:
 
 
 def map_user(kc_user) -> User:
-    user =  User.from_dict(kc_user if isinstance(kc_user, dict) else kc_user._raw_dict)
+    if isinstance(kc_user, dict):
+        raw = kc_user
+    else:
+        try:
+            raw = kc_user._raw_dict
+        except AttributeError:
+            # cloudharness model base changed the raw dict attribute name
+            raw = kc_user.var__raw_dict
+    user = User.from_dict(raw)
     if 'attributes' not in kc_user or not kc_user['attributes']:
         kc_user['attributes'] = {}
 

applications/accounts-api/backend/requirements.txt

@@ -1,11 +1,9 @@
-connexion[swagger-ui] >= 2.6.0; python_version>="3.6"
-# 2.3 is the last version that supports python 3.4-3.5
-connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4"
-# connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug
-# we must peg werkzeug versions below to fix connexion
-# https://github.com/zalando/connexion/pull/1044
-werkzeug == 2.2.3; python_version=="3.5" or python_version=="3.4"
-swagger-ui-bundle >= 0.0.2
-python_dateutil >= 2.6.0
-setuptools >= 21.0.0
-cloudharness
+connexion[swagger-ui,flask,uvicorn]>=3.0.0,<4.0.0
+swagger-ui-bundle>=1.1.0
+python_dateutil>=2.9.0
+setuptools>=21.0.0
+uvicorn
+# Following some unnecessary requirements to make sure they can be installed
+psycopg2-binary
+sqlalchemy<2.0.0
+scipy

applications/accounts-api/backend/setup.py

@@ -14,10 +14,10 @@
 # http://pypi.python.org/pypi/setuptools
 
 REQUIRES = [
-    "connexion>=2.0.2",
-    "swagger-ui-bundle>=0.0.2",
-    "python_dateutil>=2.6.0",
-    "cloudharness"
+    "connexion[swagger-ui,flask,uvicorn]>=3.0.0,<4.0.0",
+    "Flask",
+    "swagger-ui-bundle",
+    "cloudharness",
 ]
 
 setup(

applications/accounts/deploy/resources/realm.json

@@ -322,6 +322,34 @@
                     "offline_access",
                     "microprofile-jwt"
                 ]
+            },
+            {
+                "clientId": "idp-arc",
+                "name": "idp-arc",
+                "enabled": true,
+                "publicClient": true,
+                "standardFlowEnabled": true,
+                "implicitFlowEnabled": false,
+                "directAccessGrantsEnabled": false,
+                "serviceAccountsEnabled": false,
+                "protocol": "openid-connect",
+                "rootUrl": "{{ .Values.idpArc.rootUrl }}",
+                "redirectUris": [
+                    "{{ .Values.idpArc.rootUrl }}/*"
+                ],
+                "webOrigins": [
+                    "{{ .Values.idpArc.rootUrl }}"
+                ],
+                "attributes": {
+                    "pkce.code.challenge.method": "S256"
+                },
+                "fullScopeAllowed": true,
+                "defaultClientScopes": [
+                    "web-origins", "profile", "roles", "email"
+                ],
+                "optionalClientScopes": [
+                    "offline_access", "{{ .Values.apps.accounts.admin.role }}-scope"
+                ]
             }
         ],
         "clientScopes": [

applications/jupyterhub/deploy/values.yaml

@@ -30,10 +30,30 @@ harness:
     quota-storage-max : 1.25
 nfs_volumes: false
 hub:
+  # Control-plane sizing to sustain ~20 concurrent users. The hub coordinates
+  # auth/spawn and talks to the sqlite DB; without explicit resources it was
+  # CPU-throttled/OOM-killed under load, which surfaced downstream as 503s.
+  resources:
+    requests:
+      cpu: 0.1
+      memory: 1Gi
+    limits:
+      cpu: 4
+      memory: 1Gi
   config:
+    Authenticator:
+      # Users granted JupyterHub admin rights.
+      admin_users:
+        - a
+        - filippo@metacell.us
     JupyterHub:
       admin_access: true
       authenticator_class: ch
+      # Cap simultaneous spawns so a burst of logins doesn't stampede the
+      # hub/proxy; extra spawns queue instead of overwhelming the control plane.
+      concurrent_spawn_limit: 10
+      # Headroom above the 20-user target (named servers count individually).
+      active_server_limit: 40
   extraConfig:
     timing: |
       c.Spawner.port = 8000
@@ -42,6 +62,18 @@ hub:
       c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost localhost:3000 *.osb.local *.metacell.us *.opensourcebrain.org *.v2.opensourcebrain.org"}}
     spawner: >-
       c.Spawner.args = []
+proxy:
+  # configurable-http-proxy (CHP) fronts every request and returns 503 when it
+  # is starved or the hub behind it is unreachable. Give it guaranteed capacity
+  # so it stays responsive while routing ~20 concurrent users.
+  chp:
+    resources:
+      requests:
+        cpu: 0.2
+        memory: 256Mi
+      limits:
+        cpu: 1
+        memory: 512Mi
 singleuser:
   storage:
     type: dynamic

applications/jupyterlab/Dockerfile

@@ -77,15 +77,15 @@ USER root
 RUN conda install conda-forge::nest-simulator
 ENV NEST_HOME=/opt/conda
 
-USER jovyan
-
-#### Install Python packages
+#### Install Python packages (as root to avoid permission issues with conda-managed packages)
 
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip && \ 
     pip install backports.tarfile>=1.2 # temp fix for error: ImportError: cannot import name 'tarfile' from 'backports'
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ 
     pip install -r requirements.txt --upgrade --no-cache-dir
 
+USER jovyan
+
 #  Compile NEURON mod files for PyNN
 RUN cd /opt/conda/lib/python3.12/site-packages/pyNN/neuron/nmodl && nrnivmodl
 

applications/jupyterlab/requirements.txt

@@ -3,7 +3,7 @@ pyelectro
 neurotune
 
 ####  NEURON & NetPyNE
-neuron
+neuron<9
 
 # Install specific version of NetPyNE
 git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne

applications/netpyne/Dockerfile

@@ -1,6 +1,6 @@
 
 
-FROM node:18 AS jsbuild
+FROM node:16-bullseye AS jsbuild
 ARG APP_DIR=dependencies/NetPyNE-UI
 ENV FOLDER=netpyne
 

applications/netpyne/deploy/values.yaml

@@ -1,7 +1,5 @@
 harness:
   subdomain: netpyne
-  secured: true
-  uri_role_mapping: []
   service:
     auto: false
     port: 80

applications/nfsserver/deploy/values-dev.yaml

@@ -1,4 +1,4 @@
 nfs:
   useDNS: true
 server:
-  diskSize: 80Gi
+  diskSize: 160Gi

applications/nfsserver/deploy/values.yaml

@@ -2,7 +2,7 @@
 harness:
   deployment:
     auto: false
-    image: gcr.io/metacellllc/osb/nfsserver:0.7.6
+    image: gcr.io/metacellllc/cloud-harness/nfsserver:featurech-261
 # nfs server pvc disk size (/exports)
 server:
   diskSize: 240Gi

applications/osb-portal/src/components/workspace/WorkspaceFrame.tsx

@@ -3,13 +3,18 @@ import { useParams } from "react-router-dom";
 import { AnyAction, Dispatch } from "redux";
 import { PayloadAction } from "@reduxjs/toolkit";
 
+import Box from "@mui/material/Box";
+import Button from "@mui/material/Button";
+import Typography from "@mui/material/Typography";
+
 import {
   Workspace,
   WorkspaceResource,
   OSBApplications,
   ResourceStatus,
 } from "../../types/workspace";
 import { UserInfo } from "../../types/user";
+import workspaceService from "../../service/WorkspaceService";
 import WorkspaceResourceService from "../../service/WorkspaceResourceService";
 import { getBaseDomain, getApplicationDomain } from "../../utils";
 
@@ -30,6 +35,8 @@ export const WorkspaceFrame = (props: {
   currentResource: WorkspaceResource;
 }) => {
   const [frameUrl, setFrameUrl] = React.useState(null);
+  const [frameError, setFrameError] = React.useState(false);
+  const [retryToken, setRetryToken] = React.useState(0);
   const { app } = useParams<{ app: string }>();
 
   const { user, workspace, dispatch, currentResource } = props;
@@ -57,7 +64,7 @@ export const WorkspaceFrame = (props: {
       "workspace-frame"
     ) as HTMLIFrameElement;
     const messageListener = (message: MessageEvent<PayloadAction>) => {
-      if (message.source !== iFrame.contentWindow) {
+      if (!iFrame || message.source !== iFrame.contentWindow) {
         return;
       }
       console.debug("Message", message);
@@ -99,24 +106,92 @@ export const WorkspaceFrame = (props: {
   }, [openResource]);
 
   React.useEffect(() => {
-    const applicationDomain = getApplicationDomain(application);
-    const domain = getBaseDomain();
-
-    const userParam = user == null ? "" : `${user.id}`;
-    const type = application.subdomain.slice(0, 4);
-    document.cookie = `workspaceId=${workspace.id};path=/;domain=${domain}`;
-    if(applicationDomain) {
-      setFrameUrl(`//${applicationDomain}/hub/spawn/${userParam}/${workspace.id}${type}${document.location.search ?? ''}`);
-    } else {
-      setFrameUrl(`/testapp?workspaceId=${workspace.id}&type=${type}&user=${userParam}&application=${application.name}`);
-    }
+    let cancelled = false;
+    setFrameError(false);
+    setFrameUrl(null);
+
+    const spawn = async () => {
+      // Make sure the workspace volume is created and ready (bound) before we
+      // spawn the iframe, so the app pod does not fail to mount it.
+      try {
+        await workspaceService.ensureWorkspaceReady(workspace.id);
+      } catch (e) {
+        if (e?.volumeNotReady) {
+          // Transient: the storage is genuinely not bound yet. Surface a
+          // retryable error instead of spawning a pod that would fail to mount.
+          console.error("Workspace volume not ready, not spawning iframe", e);
+          if (!cancelled) {
+            setFrameError(true);
+          }
+          return;
+        }
+        // Any other failure (auth, network, 5xx) is unrelated to volume
+        // readiness; don't block workspace access — fall through and spawn the
+        // iframe as we did before this readiness check existed.
+        console.warn("Workspace readiness check failed, spawning anyway", e);
+      }
+      if (cancelled) {
+        return;
+      }
 
-  }, [application, user, workspace.id]);
+      const applicationDomain = getApplicationDomain(application);
+      const domain = getBaseDomain();
+
+      const userParam = user == null ? "" : `${user.id}`;
+      const type = application.subdomain.slice(0, 4);
+      document.cookie = `workspaceId=${workspace.id};path=/;domain=${domain}`;
+      if (applicationDomain) {
+        setFrameUrl(`//${applicationDomain}/hub/spawn/${userParam}/${workspace.id}${type}${document.location.search ?? ''}`);
+      } else {
+        setFrameUrl(`/testapp?workspaceId=${workspace.id}&type=${type}&user=${userParam}&application=${application.name}`);
+      }
+    };
+
+    spawn();
+
+    return () => {
+      cancelled = true;
+    };
+    // Depend on the primitive inputs the effect actually uses, not the
+    // `application`/`user` object identities — those are rebuilt on unrelated
+    // redux updates and would otherwise reset frameUrl to null (remounting the
+    // iframe via key={frameUrl}) and refire the /open fetch on every re-render.
+  }, [application?.subdomain, application?.name, user?.id, workspace.id, retryToken]);
 
   if (!workspace) {
     return null;
   }
 
+  if (frameError) {
+    return (
+      <Box
+        display="flex"
+        flexDirection="column"
+        alignItems="center"
+        justifyContent="center"
+        flex={1}
+        gap={2}
+        p={4}
+        textAlign="center"
+      >
+        <Typography variant="h6">
+          The workspace is temporarily unavailable
+        </Typography>
+        <Typography variant="body1" color="textSecondary">
+          The workspace storage is not ready yet. This is usually temporary —
+          please try again in a few seconds.
+        </Typography>
+        <Button
+          variant="contained"
+          color="primary"
+          onClick={() => setRetryToken((token) => token + 1)}
+        >
+          Retry
+        </Button>
+      </Box>
+    );
+  }
+
   return (
     <iframe
       id="workspace-frame"

applications/osb-portal/src/service/ErrorHandleService.tsx

@@ -1,19 +1,17 @@
 import * as Sentry from "@sentry/react";
 
 export async function initErrorHandler(appName: string) {
-  const commonUrl =
-    window.location.origin + "/proxy/common/api/sentry/getdsn/" + appName;
-  fetch(commonUrl)
-    .then(
-      (response) => response.json(),
-      (error) => console.error("Cannot connect to common service")
-    )
-    .then(
-      (sentryDSN) => {
-        if (sentryDSN) {
-          Sentry.init({ dsn: sentryDSN.dsn });
-        }
-      },
-      (error) => console.error("Cannot connect to error monitor", error)
-    );
+  const dsn = import.meta.env.VITE_SENTRY_DSN;
+  if (!dsn) {
+    // No DSN configured for this deployment: leave Sentry uninitialized.
+    return;
+  }
+
+  Sentry.init({
+    dsn,
+    sampleRate: Number(import.meta.env.VITE_SENTRY_SAMPLE_RATE),
+    tracesSampleRate: Number(import.meta.env.VITE_SENTRY_TRACES_SAMPLE_RATE),
+  });
+  // Preserve the per-app attribution the previous per-app DSN provided.
+  Sentry.setTag("app", appName);
 }