refactor: consolidate tooling

.env.template

@@ -0,0 +1,65 @@
+# RAG Template Environment Variables
+# Copy this file to .env and fill in your actual values
+
+# =============================================================================
+# S3 Object Storage Configuration (Required)
+# =============================================================================
+S3_ACCESS_KEY_ID=your_s3_access_key_here
+S3_SECRET_ACCESS_KEY=your_s3_secret_key_here
+
+# =============================================================================
+# Basic Authentication (Required)
+# =============================================================================
+BASIC_AUTH="foo:$apr1$ryE0iE7H$F2SlPDNoFdGoaHrcla2HL/"
+
+# =============================================================================
+# Langfuse Configuration (Required for observability)
+# =============================================================================
+LANGFUSE_PUBLIC_KEY=pk-lf-your-public-key-here
+LANGFUSE_SECRET_KEY=sk-lf-your-secret-key-here
+
+# Langfuse Initialization (Optional - for pre-creating organization/project)
+LANGFUSE_INIT_ORG_ID="my-org"
+LANGFUSE_INIT_PROJECT_ID="my-project"
+LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-your-public-key-here
+LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-your-secret-key-here
+LANGFUSE_INIT_USER_EMAIL="[email protected]"
+LANGFUSE_INIT_USER_NAME="stackiteer"
+LANGFUSE_INIT_USER_PASSWORD="stackit123"
+
+# =============================================================================
+# Frontend Authentication (Required)
+# =============================================================================
+VITE_AUTH_USERNAME=foo
+VITE_AUTH_PASSWORD=bar
+
+# =============================================================================
+# LLM Provider API Keys (Choose one or more)
+# =============================================================================
+
+# STACKIT AI Model Serving as provider (Recommended)
+# A guide how to get your API key by using the STACKIT portal UI: https://docs.stackit.cloud/stackit/en/getting-started-with-the-stackit-portal-ui-319914591.html
+# A guide how to get your API key by using the Product API: https://docs.stackit.cloud/stackit/en/getting-started-with-the-product-api-319914605.html
+STACKIT_VLLM_API_KEY=your-stackit-vllm-api-key
+STACKIT_EMBEDDER_API_KEY=your-stackit-embedder-api-key
+
+# OpenAI (For RAGAS evaluation - Optional)
+RAGAS_OPENAI_API_KEY=sk-your-openai-api-key-here
+
+# =============================================================================
+# Confluence Integration (Optional)
+# =============================================================================
+CONFLUENCE_TOKEN="your-confluence-api-token,your-confluence-api-token"
+CONFLUENCE_URL="https://your-company.atlassian.net,https://your-company.atlassian.net"
+CONFLUENCE_SPACE_KEY="YOUR_SPACE_KEY_1,YOUR_SPACE_KEY_2"
+CONFLUENCE_VERIFY_SSL="True,True"
+CONFLUENCE_DOCUMENT_NAME="document1,document2"
+
+# =============================================================================
+# Additional Notes:
+# =============================================================================
+# 1. Required variables must be set for the system to work properly
+# 2. S3-compatible storage is required for document storage
+# 3. Choose either STACKIT or configure Ollama for LLM providers
+# 4. Confluence integration is optional but enables document ingestion from Confluence
+# 5. For production deployment, see infrastructure/README.md for additional configuration

.github/workflows/lint-and-test.yml

@@ -1,12 +1,48 @@
 name: lint-and-test
 
 on:
-    pull_request:
-      branches:
+  pull_request:
+    branches:
       - main
-    workflow_dispatch:
+  workflow_dispatch:
+
+env:
+  NODE_VERSION: '18'
+  PYTHON_VERSION: '3.11'
 
 jobs:
+  changes:
+    name: Detect Changes
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.changes.outputs.services }}
+      libs: ${{ steps.changes.outputs.libs }}
+      frontend: ${{ steps.changes.outputs.frontend }}
+      infrastructure: ${{ steps.changes.outputs.infrastructure }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect changes
+        id: changes
+        uses: dorny/paths-filter@v2
+        with:
+          filters: |
+            services:
+              - 'services/admin-backend/**'
+              - 'services/document-extractor/**'
+              - 'services/rag-backend/**'
+              - 'services/mcp-server/**'
+            libs:
+              - 'libs/**'
+            frontend:
+              - 'services/frontend/**'
+            infrastructure:
+              - 'infrastructure/**'
+              - 'Tiltfile'
+
   sanitize-branch-name:
     runs-on: ubuntu-latest
     outputs:
@@ -15,75 +51,176 @@ jobs:
       - name: sanitize-branch-name
         id: sanitize
         run: |
-          SANITIZED_REF=$(echo "${GITHUB_HEAD_REF}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9' '-')
+          SANITIZED_REF=$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9' '-')
           SANITIZED_REF=${SANITIZED_REF#-}
           SANITIZED_REF=${SANITIZED_REF%-}
           SANITIZED_REF=${SANITIZED_REF:0:63}
           if [[ -z "$SANITIZED_REF" || "$SANITIZED_REF" =~ ^-+$ ]]; then
             SANITIZED_REF="tmp-branch"
           fi
-          echo "::set-output name=sanitized_ref::${SANITIZED_REF}"
+          echo "sanitized_ref=${SANITIZED_REF}" >> $GITHUB_OUTPUT
         shell: bash
-        env:
-          GITHUB_HEAD_REF: ${{ github.head_ref }}
 
-  update-submodules:
-    name: update-submodules
+  build-and-test-services:
+    name: Build and Test Services
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
+    needs: [changes, sanitize-branch-name]
+    if: needs.changes.outputs.services == 'true' || github.event_name == 'workflow_dispatch'
+    strategy:
+      fail-fast: false
+      matrix:
+        service: ["admin-backend", "document-extractor", "rag-backend", "mcp-server"]
     steps:
-      - name: checkout
+      - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.ref }}
-          submodules: false
 
-      - name: update-submodules
-        id: update-submodules
+      - name: Set Docker Image Name
         run: |
-          git config --global url."https://github.com/".insteadOf "[email protected]:"
-          git submodule sync --recursive
-          git submodule update --init --remote --recursive
-
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "Submodules have new commits. Committing and pushing..."
-            git config user.name "github-actions[bot]"
-            git config user.email "github-actions[bot]@users.noreply.github.com"
-            git commit -am "chore: update submodules to latest main"
-            git push
-          else
-            echo "No submodule changes."
-          fi
+          echo "IMAGE_NAME=${{ matrix.service }}:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Build Docker image
+        run: |
+          docker build -t $IMAGE_NAME --build-arg dev=1 -f services/${{ matrix.service }}/Dockerfile .
+
+      - name: Run linting
+        run: |
+          docker run --rm $IMAGE_NAME make lint
+
+      - name: Run tests
+        run: |
+          docker run --rm $IMAGE_NAME make test
 
-  build-and-lint:
-    name: Build and Lint
+  build-and-test-libs:
+    name: Build and Test Libraries
     runs-on: ubuntu-latest
-    needs: [sanitize-branch-name, update-submodules]
+    needs: [changes, sanitize-branch-name]
+    if: needs.changes.outputs.libs == 'true' || github.event_name == 'workflow_dispatch'
     strategy:
-        fail-fast: true
-        matrix:
-            service: [ "admin-backend", "document-extractor", "rag-backend"]
+      fail-fast: false
+      matrix:
+        library: ["rag-core-lib", "rag-core-api", "admin-api-lib", "extractor-api-lib"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Set Docker Image Name
+      - name: Set Docker Image Names
         run: |
-          echo "IMAGE_NAME=${{ matrix.service }}:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
+          echo "LINT_IMAGE_NAME=${{ matrix.library }}-lint:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
+          echo "TEST_IMAGE_NAME=${{ matrix.library }}-test:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
         shell: bash
 
       - name: Build lint image
         run: |
-          docker build -t $IMAGE_NAME --build-arg dev=1 --build-arg TAG=debug -f services/${{ matrix.service }}/Dockerfile .
+          docker build -t $LINT_IMAGE_NAME --build-arg TEST=0 -f libs/Dockerfile libs
 
-      - name: Generate lint report
+      - name: Run linting
         run: |
-          docker run --rm --entrypoint make "$IMAGE_NAME" lint
+          docker run --rm $LINT_IMAGE_NAME make lint
+
+      - name: Build test image
+        run: |
+          docker build -t $TEST_IMAGE_NAME --build-arg DIRECTORY=${{ matrix.library }} --build-arg TEST=1 -f libs/Dockerfile libs
 
       - name: Run tests
         run: |
-          docker run --rm --entrypoint make "$IMAGE_NAME" test
+          docker run --rm $TEST_IMAGE_NAME make test
+
+  build-and-test-frontend:
+    name: Build and Test Frontend
+    runs-on: ubuntu-latest
+    needs: [changes]
+    if: needs.changes.outputs.frontend == 'true' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+          cache-dependency-path: services/frontend/package-lock.json
+
+      - name: Install dependencies
+        working-directory: services/frontend
+        run: npm ci
+
+      - name: Run linting
+        working-directory: services/frontend
+        run: npm run eslint
+
+      - name: Run tests
+        working-directory: services/frontend
+        run: npm run test
+
+      - name: Build chat app
+        working-directory: services/frontend
+        run: npm run chat:build
+
+      - name: Build admin app
+        working-directory: services/frontend
+        run: npm run admin:build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: frontend-build-artifacts
+          path: |
+            services/frontend/apps/*/dist/
+          retention-days: 7
+
+  validate-infrastructure:
+    name: Validate Infrastructure
+    runs-on: ubuntu-latest
+    needs: [changes]
+    if: needs.changes.outputs.infrastructure == 'true' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: "~1.0"
+
+      - name: Terraform Format Check
+        working-directory: infrastructure/terraform
+        run: terraform fmt -check -recursive
+
+      - name: Terraform Init
+        working-directory: infrastructure/terraform
+        run: terraform init -backend=false
+
+      - name: Terraform Validate
+        working-directory: infrastructure/terraform
+        run: terraform validate
+
+      - name: Validate Helm Charts
+        run: |
+          curl https://get.helm.sh/helm-v3.12.0-linux-amd64.tar.gz | tar xz
+          sudo mv linux-amd64/helm /usr/local/bin/helm
+          helm lint infrastructure/rag/
+
+  summary:
+    name: CI Summary
+    runs-on: ubuntu-latest
+    needs: [build-and-test-services, build-and-test-libs, build-and-test-frontend, validate-infrastructure]
+    if: always()
+    steps:
+      - name: Check results
+        run: |
+          echo "Services: ${{ needs.build-and-test-services.result }}"
+          echo "Libraries: ${{ needs.build-and-test-libs.result }}"
+          echo "Frontend: ${{ needs.build-and-test-frontend.result }}"
+          echo "Infrastructure: ${{ needs.validate-infrastructure.result }}"
+
+          if [[ "${{ needs.build-and-test-services.result }}" == "failure" ]] || \
+             [[ "${{ needs.build-and-test-libs.result }}" == "failure" ]] || \
+             [[ "${{ needs.build-and-test-frontend.result }}" == "failure" ]] || \
+             [[ "${{ needs.validate-infrastructure.result }}" == "failure" ]]; then
+            echo "❌ One or more jobs failed"
+            exit 1
+          else
+            echo "✅ All jobs passed or were skipped"
+          fi