github test dorny/test-reporter

.github/workflows/make-and-test.yml

@@ -1,4 +1,4 @@
-name: Make all packages and run their tests
+name: Test
 
 # newer commits in the same PR abort running ones for the same workflow
 concurrency:
@@ -43,7 +43,7 @@ jobs:
       - name: Check doc links
         run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py
 
-  build:
+  test:
     runs-on: ubuntu-latest
 
     services:
@@ -94,7 +94,7 @@ jobs:
       - name: Download and install Valkey
         run: |
           VALKEY_VERSION=8.1.2
-          curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
+          curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
           tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
           sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/
 
@@ -106,6 +106,54 @@ jobs:
           pip install ipykernel
           python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)"
 
-      - run: cd src && npm install -g pnpm
-      - run: cd src && pnpm run make
-      - run: source venv/bin/activate && cd src && pnpm run test-github-ci
+
+      - name: install pnpm
+        #run: cd src && npm install -g pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+          run_install: false
+
+      - name: build
+        run: cd src && pnpm run make
+
+      - name: test
+        run: source venv/bin/activate && cd src && pnpm run test-github-ci:report
+
+      - name: upload test results
+        uses: actions/upload-artifact@v4  # upload test results
+        if: ${{ !cancelled() }}           # run this step even if previous step failed
+        with:
+          name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
+          path: 'src/packages/*/junit.xml'
+
+      - name: dependency check
+        run: cd src && pnpm run depcheck
+
+  report:
+    runs-on: ubuntu-latest
+
+    needs: [test]
+
+    if: ${{ !cancelled() }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download all test artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "test-results-*"
+          merge-multiple: true
+          path: test-results/
+
+      - name: Test Report
+        uses: dorny/test-reporter@v2
+        with:
+          name: CoCalc Jest Tests
+          path: 'test-results/**/junit.xml'
+          reporter: jest-junit
+          use-actions-summary: 'true'
+          fail-on-error: false
+

.gitignore

@@ -159,3 +159,6 @@ src/packages/frontend/i18n/trans/*.compiled.json
 **/*.db
 **/project-env.sh
 **/*.bash_history
+
+# test reports by jest-junit
+junit.xml

src/CLAUDE.md

@@ -1,18 +1,19 @@
-# CLAUDE.md
+# CLAUDE.md and GEMINI.md
 
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+This file provides guidance to Claude Code (claude.ai/code) and also Gemini CLI (https://github.com/google-gemini/gemini-cli) when working with code in this repository.
 
 # CoCalc Source Repository
 
-* This is the source code of CoCalc in a Git repository
-* It is a complex JavaScript/TypeScript SaaS application
-* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
-* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
+- This is the source code of CoCalc in a Git repository
+- It is a complex JavaScript/TypeScript SaaS application
+- CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
+- The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
 
 ## Code Style
 
 - Everything is written in TypeScript code
 - Indentation: 2-spaces
+- Run `pretter -w [filename]` after modifying a file (ts, tsx, md, json, ...) to format it correctly.
 - All .js and .ts files are formatted by the tool prettier
 - Add suitable types when you write code
 - Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style.
@@ -23,28 +24,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 ## Development Commands
 
 ### Essential Commands
+
 - `pnpm build-dev` - Build all packages for development
 - `pnpm clean` - Clean all node_modules and dist directories
-- `pnpm database` - Start PostgreSQL database server
-- `pnpm hub` - Start the main hub server
-- `pnpm psql` - Connect to the PostgreSQL database
 - `pnpm test` - Run full test suite
-- `pnpm test-parallel` - Run tests in parallel across packages
 - `pnpm depcheck` - Check for dependency issues
+- `prettier -w [filename]` to format the style of a file after editing it
+- after creating a file, run `git add [filename]` to start tracking it
 
 ### Package-Specific Commands
-- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package
+
+- `cd packages/[package] && pnpm build` - Build and compile a specific package
+  - for packages/next and packages/static, run `cd packages/[package] && pnpm build-dev`
+- `cd packages/[package] && pnpm tsc:watch` - TypeScript compilation in watch mode for a specific package
 - `cd packages/[package] && pnpm test` - Run tests for a specific package
 - `cd packages/[package] && pnpm build` - Build a specific package
+- **IMPORTANT**: When modifying packages like `util` that other packages depend on, you must run `pnpm build` in the modified package before typechecking dependent packages
 
-### Development Setup
-1. Start database: `pnpm database`
-2. Start hub: `pnpm hub`
-3. For TypeScript changes, run `pnpm tsc` in the relevant package directory
+### Development
+
+- **IMPORTANT**: Always run `prettier -w [filename]` immediately after editing any .ts, .tsx, .md, or .json file to ensure consistent styling
+- After TypeScript or `*.tsx` changes, run `pnpm build` in the relevant package directory
 
 ## Architecture Overview
 
 ### Package Structure
+
 CoCalc is organized as a monorepo with key packages:
 
 - **frontend** - React/TypeScript frontend application using Redux-style stores and actions
@@ -62,25 +67,30 @@ CoCalc is organized as a monorepo with key packages:
 ### Key Architectural Patterns
 
 #### Frontend Architecture
+
 - **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`)
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
 
 #### Backend Architecture
+
 - **PostgreSQL Database**: Primary data store with sophisticated querying system
 - **WebSocket Messaging**: Real-time communication between frontend and backend
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
 
 #### Communication Patterns
+
 - **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`)
 - **Database Queries**: Structured query system with typed interfaces
 - **Event Emitters**: Inter-service communication within backend
 - **REST-like APIs**: Some HTTP endpoints for specific operations
+- **API Schema**: API endpoints in `packages/next/pages/api/v2/` use Zod schemas in `packages/next/lib/api/schema/` for validation. These schemas must be kept in harmony with the TypeScript types sent from frontend applications using `apiPost` (in `packages/next/lib/api/post.ts`) or `api` (in `packages/frontend/client/api.ts`). When adding new fields to API requests, both the frontend types and the API schema validation must be updated.
 
 ### Key Technologies
+
 - **TypeScript**: Primary language for all new code
 - **React**: Frontend framework
 - **PostgreSQL**: Database
@@ -91,40 +101,56 @@ CoCalc is organized as a monorepo with key packages:
 - **SASS**: CSS preprocessing
 
 ### Database Schema
+
 - Comprehensive schema in `packages/util/db-schema`
 - Query abstractions in `packages/database/postgres/`
 - Type-safe database operations with TypeScript interfaces
 
 ### Testing
+
 - **Jest**: Primary testing framework
 - **ts-jest**: TypeScript support for Jest
 - **jsdom**: Browser environment simulation for frontend tests
 - Test files use `.test.ts` or `.spec.ts` extensions
 - Each package has its own jest.config.js
 
 ### Import Patterns
+
 - Use absolute imports with `@cocalc/` prefix for cross-package imports
 - Example: `import { cmp } from "@cocalc/util/misc"`
 - Type imports: `import type { Foo } from "./bar"`
 - Destructure imports when possible
 
 ### Development Workflow
-1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package)
+
+1. Changes to TypeScript require compilation (`pnpm build` in relevant package)
 2. Database must be running before starting hub
 3. Hub coordinates all services and should be restarted after changes
 4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes
 
 # Workflow
-- Be sure to typecheck when you're done making a series of code changes
+
+- Be sure to build when you're done making a series of code changes
 - Prefer running single tests, and not the whole test suite, for performance
 
 ## Git Workflow
 
+- Never modify a file when in the `master` or `main` branch
+- All changes happen through feature branches, which are pushed as pull requests to GitHub
+- When creating a new file, run `git add [filename]` to track the file.
 - Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
 - When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
 
-# important-instruction-reminders
+# Important Instruction Reminders
+
 - Do what has been asked; nothing more, nothing less.
 - NEVER create files unless they're absolutely necessary for achieving your goal.
 - ALWAYS prefer editing an existing file to creating a new one.
-- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
+- REFUSE to modify files when the git repository is on the `master` or `main` branch.
+- NEVER proactively create documentation files (`*.md`) or README files. Only create documentation files if explicitly requested by the User.
+
+# Ignore
+
+- Ignore files covered by `.gitignore`
+- Ignore everything in `node_modules` or `dist` directories
+- Ignore all files not tracked by Git, unless they are newly created files

src/GEMINI.md

@@ -0,0 +1 @@
+CLAUDE.md

src/package.json

@@ -19,6 +19,7 @@
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test",
     "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1",
+    "test-github-ci:report": "pnpm run test-github-ci --report",
     "depcheck": "cd packages && pnpm run -r --parallel depcheck",
     "prettier-all": "cd packages/",
     "local-ci": "./scripts/ci.sh",

src/packages/assets/package.json

@@ -11,6 +11,7 @@
     "preinstall": "npx only-allow pnpm",
     "build": "./build.sh",
     "test": "../node_modules/.bin/jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "prepublishOnly": "pnpm test"
   },
   "keywords": ["cocalc"],

src/packages/backend/package.json

@@ -10,14 +10,19 @@
     "./auth/*": "./dist/auth/*.js",
     "./auth/tokens/*": "./dist/auth/tokens/*.js"
   },
-  "keywords": ["utilities", "cocalc"],
+  "keywords": [
+    "utilities",
+    "cocalc"
+  ],
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "clean": "rm -rf dist node_modules",
     "build": "pnpm exec tsc --build",
     "tsc": "pnpm exec tsc --watch  --pretty --preserveWatchOutput",
     "test": "pnpm exec jest --forceExit",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "test-conat": " pnpm exec jest --forceExit conat",
+    "test-conat:report": " pnpm exec jest --forceExit conat --reporters=default --reporters=jest-junit",
     "testp": "pnpm exec jest --forceExit",
     "depcheck": "pnpx depcheck --ignores events",
     "prepublishOnly": "pnpm test",
@@ -28,7 +33,12 @@
     "conat-persist": "DEBUG=cocalc:*  node ./bin/conat-persist.cjs",
     "conat-test-server": "node ./bin/conat-test-server.cjs"
   },
-  "files": ["dist/**", "bin/**", "README.md", "package.json"],
+  "files": [
+    "dist/**",
+    "bin/**",
+    "README.md",
+    "package.json"
+  ],
   "author": "SageMath, Inc.",
   "license": "SEE LICENSE.md",
   "dependencies": {
@@ -57,6 +67,7 @@
   },
   "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend",
   "devDependencies": {
-    "@types/node": "^18.16.14"
+    "@types/node": "^18.16.14",
+    "jest-junit": "^16.0.0"
   }
 }

src/packages/cdn/package.json

@@ -16,6 +16,7 @@
     "build": "rm -rf build dist && mkdir build && cp -rv *.json *.py *.js build/ && cd build && npm ci && mkdir dist && ./setup.py && mv dist .. && cd .. && rm -rf build ",
     "postinstall": "./postinstall.js",
     "test": "../node_modules/.bin/jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "prepublishOnly": "npm test"
   },
   "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/cdn",

src/packages/conat/package.json

@@ -22,6 +22,7 @@
     "clean": "rm -rf dist node_modules",
     "tsc": "pnpm exec tsc  --watch  --pretty --preserveWatchOutput",
     "test": "pnpm exec jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck --ignores events"
   },
   "files": [

src/packages/database/package.json

@@ -38,14 +38,16 @@
     "@types/lodash": "^4.14.202",
     "@types/node": "^18.16.14",
     "@types/pg": "^8.6.1",
-    "coffeescript": "^2.5.1"
+    "coffeescript": "^2.5.1",
+    "jest-junit": "^16.0.0"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "build": "../node_modules/.bin/tsc  --build && coffee -c -o dist/ ./",
     "clean": "rm -rf dist",
     "tsc": "../node_modules/.bin/tsc --watch  --pretty --preserveWatchOutput",
     "test": "pnpm exec jest --forceExit --runInBand",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'",
     "prepublishOnly": "pnpm test"
   },

src/packages/file-server/package.json

@@ -11,6 +11,7 @@
     "build": "pnpm exec tsc --build",
     "tsc": "pnpm exec tsc  --watch  --pretty --preserveWatchOutput",
     "test": "pnpm exec jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck",
     "clean": "rm -rf node_modules dist"
   },

src/packages/frontend/package.json

@@ -175,6 +175,7 @@
     "build-coffee": "pnpm exec coffee -m -c -o dist/ .",
     "tsc": "NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ",
     "test": "jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck --ignores cspell,@cspell/dict-typescript,events,@formatjs/cli | grep -Ev '\\.coffee|coffee$'",
     "prepublishOnly": "pnpm test",
     "update-color-scheme": "node ./update-color-scheme.js",

src/packages/hub/package.json

@@ -49,7 +49,8 @@
   "devDependencies": {
     "@types/express": "^4.17.21",
     "@types/node": "^18.16.14",
-    "coffeescript": "^2.5.1"
+    "coffeescript": "^2.5.1",
+    "jest-junit": "^16.0.0"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -64,6 +65,8 @@
     "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem",
     "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0",
     "tsc": "tsc --watch  --pretty --preserveWatchOutput",
+    "test": "jest dist/",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'",
     "prepublishOnly": "test"
   },

src/packages/jupyter/package.json

@@ -24,6 +24,7 @@
     "build": "../node_modules/.bin/tsc --build",
     "clean": "rm -rf node_modules dist",
     "test": "pnpm exec jest --forceExit --maxWorkers=1",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "depcheck": "pnpx depcheck",
     "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput"
   },

src/packages/next/package.json

@@ -36,6 +36,7 @@
     "start": "unset PGHOST; pnpm exec next start",
     "start-project": "unset PGHOST PGUSER COCALC_ROOT; export PORT=5000 BASE_PATH=/$COCALC_PROJECT_ID/port/5000; echo https://cocalc.com$BASE_PATH; pnpm start",
     "test": "NODE_ENV='dev' pnpm exec jest",
+    "test:report": "pnpm run test -- --reporters=default --reporters=jest-junit",
     "test-api": "NODE_ENV='production' pnpm exec jest ./lib/api/framework.test.ts",
     "depcheck": "pnpx depcheck --ignores @openapitools/openapi-generator-cli,eslint-config-next,locales,components,lib,public,pages,software-inventory,pg",
     "prepublishOnly": "pnpm test",