Add SpeziLLMFog that performs dynamic LLM job dispatch to fog nodes (#52

)

# Add SpeziLLMFog that performs dynamic LLM job dispatch to fog nodes

## ♻️ Current situation & Problem
Currently, SpeziLLM doesn't support a privacy friendly execution of LLM
inference jobs within the local network.


## ⚙️ Release Notes 
- Add SpeziLLMFog that performs dynamic LLM job dispatch to fog nodes.
- Include a FogNode component that easily starts up a complete fog node
ready to perform LLM inference requests via docker compose.
- Introduce custom `LLMContext` type that holds the internal state of
the `LLMSession`, instead of relying on the `SpeziChat` models.


## 📚 Documentation
Added proper readmes, docs, and in-line comments


## ✅ Testing
Manual testing (for now)


## 📝 Code of Conduct & Contributing Guidelines 

By submitting creating this pull request, you agree to follow our [Code
of
Conduct](https://github.com/StanfordSpezi/.github/blob/main/CODE_OF_CONDUCT.md)
and [Contributing
Guidelines](https://github.com/StanfordSpezi/.github/blob/main/CONTRIBUTING.md):
- [x] I agree to follow the [Code of
Conduct](https://github.com/StanfordSpezi/.github/blob/main/CODE_OF_CONDUCT.md)
and [Contributing
Guidelines](https://github.com/StanfordSpezi/.github/blob/main/CONTRIBUTING.md).

.github/workflows/build-and-test.yml

@@ -1,7 +1,7 @@
 #
 # This source file is part of the Stanford Spezi open source project
 #
-# SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+# SPDX-FileCopyrightText: 2023 Stanford University and the project authors (see CONTRIBUTORS.md)
 #
 # SPDX-License-Identifier: MIT
 #
@@ -16,40 +16,124 @@ on:
   workflow_dispatch:
 
 jobs:
-  buildandtest:
-    name: Build and Test Swift Package
+  buildandtest_ios:
+    name: Build and Test Swift Package iOS
     uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
     strategy:
       matrix:
         include:
           - buildConfig: Debug
-            artifactname: SpeziLLM-Package.xcresult
+            artifactname: SpeziLLM-iOS.xcresult
+            resultBundle: SpeziLLM-iOS.xcresult
           - buildConfig: Release
-            artifactname: SpeziLLM-Package-Release.xcresult
+            artifactname: SpeziLLM-iOS-Release.xcresult
+            resultBundle: SpeziLLM-iOS-Release.xcresult
     with:
+      runsonlabels: '["macOS", "self-hosted"]'
+      scheme: SpeziLLM-Package
+      buildConfig: ${{ matrix.buildConfig }}
+      resultBundle: ${{ matrix.resultBundle }}
       artifactname: ${{ matrix.artifactname }}
+  buildandtest_visionos:
+    name: Build and Test Swift Package visionOS
+    uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
+    strategy:
+      matrix:
+        include:
+          - buildConfig: Debug
+            artifactname: SpeziLLM-visionOS.xcresult
+            resultBundle: SpeziLLM-visionOS.xcresult
+          - buildConfig: Release
+            artifactname: SpeziLLM-visionOS-Release.xcresult
+            resultBundle: SpeziLLM-visionOS-Release.xcresult
+    with:
       runsonlabels: '["macOS", "self-hosted"]'
       scheme: SpeziLLM-Package
+      destination: 'platform=visionOS Simulator,name=Apple Vision Pro'
+      buildConfig: ${{ matrix.buildConfig }}
+      resultBundle: ${{ matrix.resultBundle }}
+      artifactname: ${{ matrix.artifactname }}
+  buildandtest_macos:
+    name: Build and Test Swift Package macOS
+    uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
+    strategy:
+      matrix:
+        include:
+          - buildConfig: Debug
+            artifactname: SpeziLLM-macOS.xcresult
+            resultBundle: SpeziLLM-macOS.xcresult
+          - buildConfig: Release
+            artifactname: SpeziLLM-macOS-Release.xcresult
+            resultBundle: SpeziLLM-macOS-Release.xcresult
+    with:
+      runsonlabels: '["macOS", "self-hosted"]'
+      scheme: SpeziLLM-Package
+      destination: 'platform=macOS,arch=arm64'
+      buildConfig: ${{ matrix.buildConfig }}
+      resultBundle: ${{ matrix.resultBundle }}
+      artifactname: ${{ matrix.artifactname }}
+  buildandtestuitests_ios:
+    name: Build and Test UI Tests iOS
+    uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
+    strategy:
+      matrix:
+        include:
+          - buildConfig: Debug
+            resultBundle: TestApp-iOS.xcresult
+            artifactname: TestApp-iOS.xcresult
+          - buildConfig: Release
+            resultBundle: TestApp-iOS-Release.xcresult
+            artifactname: TestApp-iOS-Release.xcresult
+    with:
+      runsonlabels: '["macOS", "self-hosted"]'
+      path: 'Tests/UITests'
+      scheme: TestApp
       buildConfig: ${{ matrix.buildConfig }}
-  buildandtestuitests:
-    name: Build and Test UI Tests
+      resultBundle: ${{ matrix.resultBundle }}
+      artifactname: ${{ matrix.artifactname }}
+  buildandtestuitests_ipad:
+    name: Build and Test UI Tests iPadOS
     uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
     strategy:
       matrix:
         include:
           - buildConfig: Debug
-            artifactname: TestApp.xcresult
+            resultBundle: TestApp-iPad.xcresult
+            artifactname: TestApp-iPad.xcresult
           - buildConfig: Release
-            artifactname: TestApp-Release.xcresult
+            resultBundle: TestApp-iPad-Release.xcresult
+            artifactname: TestApp-iPad-Release.xcresult
     with:
+      runsonlabels: '["macOS", "self-hosted"]'
+      path: 'Tests/UITests'
+      scheme: TestApp
+      destination: 'platform=iOS Simulator,name=iPad Air (5th generation)'
+      buildConfig: ${{ matrix.buildConfig }}
+      resultBundle: ${{ matrix.resultBundle }}
       artifactname: ${{ matrix.artifactname }}
+  buildandtestuitests_visionos:
+    name: Build and Test UI Tests visionOS
+    uses: StanfordSpezi/.github/.github/workflows/xcodebuild-or-fastlane.yml@v2
+    strategy:
+      matrix:
+        include:
+          - buildConfig: Debug
+            resultBundle: TestApp-visionOS.xcresult
+            artifactname: TestApp-visionOS.xcresult
+          - buildConfig: Release
+            resultBundle: TestApp-visionOS-Release.xcresult
+            artifactname: TestApp-visionOS-Release.xcresult
+    with:
       runsonlabels: '["macOS", "self-hosted"]'
       path: 'Tests/UITests'
       scheme: TestApp
+      destination: 'platform=visionOS Simulator,name=Apple Vision Pro'
       buildConfig: ${{ matrix.buildConfig }}
+      resultBundle: ${{ matrix.resultBundle }}
+      artifactname: ${{ matrix.artifactname }}
   uploadcoveragereport:
     name: Upload Coverage Report
-    needs: [buildandtest, buildandtestuitests]
+    needs: [buildandtest_ios, buildandtest_visionos, buildandtest_macos, buildandtestuitests_ios, buildandtestuitests_ipad, buildandtestuitests_visionos]
     uses: StanfordSpezi/.github/.github/workflows/create-and-upload-coverage-report.yml@v2
     with:
-      coveragereports: SpeziLLM-Package.xcresult TestApp.xcresult
+      coveragereports: 'SpeziLLM-iOS.xcresult SpeziLLM-visionOS.xcresult SpeziLLM-macOS.xcresult TestApp-iOS.xcresult TestApp-iPad.xcresult TestApp-visionOS.xcresult'

.gitignore

@@ -32,3 +32,9 @@ docs/
 
 # UITests Project
 !UITests.xcodeproj
+
+# Generated CA and TLS keys of the Fog webservice
+*.crt
+*.key
+*.srl
+*.csr

FogNode/README.md

@@ -0,0 +1,55 @@
+<!--
+                  
+This source file is part of the Stanford Spezi open source project
+
+SPDX-FileCopyrightText: 2023 Stanford University and the project authors (see CONTRIBUTORS.md)
+
+SPDX-License-Identifier: MIT
+             
+-->
+
+# SpeziLLMFog FogNode
+
+Offers the functionality to dynamically dispatch LLM inference jobs from mobile devices to fog nodes within the local network that implement the OpenAI API.
+
+## Overview
+
+The client-side implementation of the fog execution environment is part of the Swift-based `SpeziLLM` package, specifically the [`SpeziLLMFog` target](https://swiftpackageindex.com/StanfordSpezi/SpeziLLM/documentation/spezillmfog).
+On the other hand, the server-side implementation is a web service that offers LLM inference capabilities within the local network. This setup tutorial demonstrates how to set up the server-side fog node.
+
+## Architecture
+
+The SpeziLLM fog node offers LLM inference capabilities within the local network.
+It consists of the following components:
+
+- **LLM Inference capabilities**: The LLM inference is performed by the [Ollama open-source framework](https://github.com/ollama/ollama) that executes openly available LLMs such as [Llama2](https://ollama.com/library/llama2) or [Gemma](https://ollama.com/library/gemma). A full list of the available models can be found [here](https://ollama.com/library). The API surface of [Ollama mirrors the OpenAI API](https://github.com/ollama/ollama/blob/main/docs/openai.md), at least for basic inference requests.
+- **Service advertisement**: As SpeziLLM intends to operate within a [fog computing environment](https://en.wikipedia.org/wiki/Fog_computing), the LLM execution resource (the LLM webservice) is advertised within the local network via [mDNS](https://en.wikipedia.org/wiki/Multicast_DNS). On macOS, this is achieved via [Apple's Bonjour framework](https://developer.apple.com/bonjour), on Linux the [Avahi component](https://avahi.org/) is used (both of these services are interoperable with another). 
+- **Secure local connections**: To ensure secure data transfer from and to the fog node within the local network, the [`traefik` reverse proxy](https://traefik.io/traefik/) only serves the LLM inference API via secure SSL connections. The TLS verification is achieved via a custom-issued [root CA certificate](https://en.wikipedia.org/wiki/Root_certificate) that signs the TLS certificate of the web service offering the LLM inference jobs. As these certificates need to be unique and secret, they are not part of the FogNode package but are rather generated by a script on the respective computing resource by the administrator (see setup instructions below)
+- **User authentication**: The fog node requires user authentication by verifying the passed [Firebase User ID Bearer token](https://firebase.google.com/docs/auth/admin/verify-id-tokens) in the HTTP Authentication header. By default, the fog node only verifies the authenticity of the User ID token, not if the user is actually allowed to access the resource (this could be achieved by, e.g., custom claims in the token).
+- **Packaging**: Lastly, as the fog node should be able to run on diverse platforms and the setup process should be as easy as possible, the entire fog component is packaged via [Docker](https://www.docker.com/).
+
+## Setup
+
+In order to correctly set up the Fog node, a couple of setup steps have to been taken. These steps are performed via the `setup.sh` shell script.
+
+### Requirements
+
+- Operating system: Either Linux or macOS
+- [Docker Engine v25.0](https://docs.docker.com/engine/install/) as well as [Docker Compose v2](https://docs.docker.com/compose/install/)
+- On macOS, one needs to use [Bonjour](https://developer.apple.com/bonjour) for mDNS advertisements (as Avahi only works on Linux distributions)
+
+### Executing the setup script
+
+The `setup.sh` script generates the custom CA root certificate as well as the web service certificate. They are persisted in the `ca` as well as `webservice` directories. Keep in mind that the application using the Fog Node (most likely via `SpeziLLMFog`) must trust this custom root CA certificate.
+Once the script ran through, the last output of the script should be a warning about issuing the Firebase service account key via the Firebase console. Put the file within the `auth` directory under the name `serviceAccountKey.json`, it is then automatically picked up by the authorization service.
+
+Lastly, start the container services via Docker Compose:
+- On Linux, execute `docker compose --platform=linux up` to start the service, use the `-d` flag to run it in the background like: `docker compose --platform=linux up -d`. The service is automatically advertised by Avahi via mDNS from the Docker service.
+- On macOS, run `docker compose up` to start the service. In addition, because of technical limitations of Avahi within a Docker container on macOS, one has to manually run the mDNS advertisement via Bonjour: `dns-sd -R "SpeziLLMFog Service" _https._tcp spezillmfog.local 443`. It advertises the service under the `spezillmfog.local` domain name with the `"SpeziLLMFog Service"` user-friendly name.
+
+### Development
+
+For development purposes, the `docker-compose.dev.yml` file starts up the fog node without TLS certificates and with the usage of the Firebase Emulator. In that case, one doesn't have to execute the setup script mentioned above (as no certificates are required without a TLS connection) and doesn't have to get the Firebase service account key from the Firebase Console.
+In addition, this development compose file doesn't include an mDNS advertisement service. The developer is responsible for advertising the service. On macOS, which is the primary development environment for SpeziLLMFog, this can be done via Bonjour and the `dns-sd -R "SpeziLLMFog Service" _http._tcp spezillmfog.local 80` command. Note that the service advertises an `http` service with port 80, in contrast to the production setup with HTTPS and port 443 (secure traffic).
+
+Another file for development purposes is the `docker-compose.avahi.yml` file. One container advertises an mDNS service via Avahi, another container discovers this service via an Avahi Sidecar. This setup is incredibly useful to test mDNS announcements on the Linux platform.

FogNode/auth/.gitignore

@@ -0,0 +1,54 @@
+#
+# This source file is part of the Stanford Spezi open source project
+#
+# SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+#
+# SPDX-License-Identifier: MIT
+# 
+
+# Firebase Admin SDK Service Account Key
+serviceAccountKey.json
+
+# Compiled TS project
+/dist
+# NPM dependencies
+/node_modules
+
+# IDE
+.vscode/
+.idea/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Log files
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (<https://gruntjs.com/creating-plugins#storing-task-files>)
+.grunt
+
+# Bower dependency directory (<https://bower.io/>)
+bower_components
+
+# Dependency directory
+# Commenting this out is preferred by some developers, npm can
+# handle it properly when it's symlinked (npm v3+)
+# node_modules/
+
+# TSD Debug info
+tsd-debug.log

FogNode/auth/Dockerfile

@@ -0,0 +1,41 @@
+#
+# This source file is part of the Stanford Spezi open source project
+#
+# SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+#
+# SPDX-License-Identifier: MIT
+#
+
+# Stage 1: Build stage
+FROM node:21-alpine3.19 AS builder
+
+LABEL org.opencontainers.image.authors="Philipp Zagar <[email protected]>" \
+      org.opencontainers.image.version="0.1" \
+      org.opencontainers.image.title="stanfordspezi/firebase-auth-service" \
+      org.opencontainers.image.description="SpeziLLMFog Firebase Authentication Service" \
+      org.opencontainers.image.url="https://ghcr.io/stanfordspezi/firebase-auth-service" \
+      org.opencontainers.image.source="https://github.com/StanfordSpezi/SpeziLLM"
+
+WORKDIR /usr/src/app
+
+# Install npm dependencies
+COPY package*.json ./
+RUN npm install
+
+# Copy source code and compile TypeScript project
+COPY tsconfig.json ./
+COPY src/ ./src
+RUN npm run build
+
+# Stage 2: Runtime stage
+FROM node:21-alpine3.19
+
+WORKDIR /usr/src/app
+
+# Copy compiled files and necessary npm packages from the builder stage
+COPY --from=builder /usr/src/app/dist ./dist
+COPY --from=builder /usr/src/app/node_modules ./node_modules
+COPY --from=builder /usr/src/app/package.json ./package.json
+
+# Start the nodeJS application
+CMD [ "node", "dist/index.js" ]

FogNode/auth/firebaseEmulator/.firebaserc

@@ -0,0 +1,5 @@
+{
+    "projects": {
+        "default": "spezillmfog"
+    }
+}

FogNode/auth/firebaseEmulator/.firebaserc.license

@@ -0,0 +1,5 @@
+This source file is part of the Stanford Spezi open-source project
+
+SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+
+SPDX-License-Identifier: MIT

FogNode/auth/firebaseEmulator/Dockerfile

@@ -0,0 +1,31 @@
+#
+# This source file is part of the Stanford Spezi open source project
+#
+# SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+#
+# SPDX-License-Identifier: MIT
+#
+
+FROM alpine:3.19
+
+LABEL org.opencontainers.image.authors="Philipp Zagar <[email protected]>" \
+      org.opencontainers.image.version="0.1" \
+      org.opencontainers.image.title="stanfordspezi/firebase-emulator-auth" \
+      org.opencontainers.image.description="SpeziLLMFog Firebase Emulator Auth" \
+      org.opencontainers.image.url="https://ghcr.io/stanfordspezi/firebase-emulator-auth" \
+      org.opencontainers.image.source="https://github.com/StanfordSpezi/SpeziLLM"
+
+# Install Firebase CLI
+RUN npm install -g firebase-tools
+
+WORKDIR /app
+
+# Copy firebase emulator config files
+COPY .firebaserc .firebaserc
+COPY firebase.json firebase.json
+
+# Expose web ui and auth service
+EXPOSE 4000 9099
+
+# Run the Firebase Emulators
+CMD ["firebase", "emulators:start"]

FogNode/auth/firebaseEmulator/firebase.json

@@ -0,0 +1,14 @@
+{
+    "emulators": {
+        "auth": {
+            "port": 9099,
+            "host": "0.0.0.0"
+        },
+        "ui": {
+            "enabled": true,
+            "port": 4000,
+            "host": "0.0.0.0"
+        },
+        "singleProjectMode": true
+    }
+}

FogNode/auth/firebaseEmulator/firebase.json.license

@@ -0,0 +1,5 @@
+This source file is part of the Stanford Spezi open-source project
+
+SPDX-FileCopyrightText: 2022 Stanford University and the project authors (see CONTRIBUTORS.md)
+
+SPDX-License-Identifier: MIT