diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index b233bf2421..088354e702 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -1,4 +1,4 @@ -name: Make all packages and run their tests +name: Test # newer commits in the same PR abort running ones for the same workflow concurrency: @@ -43,7 +43,7 @@ jobs: - name: Check doc links run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py - build: + test: runs-on: ubuntu-latest services: @@ -94,7 +94,7 @@ jobs: - name: Download and install Valkey run: | VALKEY_VERSION=8.1.2 - curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz + curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ @@ -106,6 +106,52 @@ jobs: pip install ipykernel python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" - - run: cd src && npm install -g pnpm - - run: cd src && pnpm run make - - run: source venv/bin/activate && cd src && pnpm run test-github-ci + + - name: install pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + run_install: false + + - name: build + run: cd src && pnpm run make + + # This runs all the tests with text output and jest-junit reporters to generate junit.xml reports + # That test-github-ci target runs workspaces.py for testing, which in turn runs the depchecks + - name: test + run: source venv/bin/activate && cd src && pnpm run test-github-ci --report + + - name: upload test results + uses: actions/upload-artifact@v4 # upload test results + if: ${{ !cancelled() }} # run this step even if previous step failed + with: + name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}" + path: 'src/packages/*/junit.xml' + + report: + runs-on: ubuntu-latest + + needs: [test] + + if: ${{ !cancelled() }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all test artifacts + uses: actions/download-artifact@v4 + with: + pattern: "test-results-*" + merge-multiple: true + path: test-results/ + + - name: Test Report + uses: dorny/test-reporter@v2 + with: + name: CoCalc Jest Tests + path: 'test-results/**/junit.xml' + reporter: jest-junit + use-actions-summary: 'true' + fail-on-error: false + diff --git a/.gitignore b/.gitignore index 7f06ad5a5c..331ad90a59 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ src/packages/frontend/i18n/trans/*.compiled.json **/*.db **/project-env.sh **/*.bash_history + +# test reports by jest-junit +junit.xml diff --git a/src/CLAUDE.md b/src/CLAUDE.md index a816597ef0..d530f9f0f4 100644 --- a/src/CLAUDE.md +++ b/src/CLAUDE.md @@ -1,18 +1,19 @@ -# CLAUDE.md +# CLAUDE.md and GEMINI.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides guidance to Claude Code (claude.ai/code) and also Gemini CLI (https://github.com/google-gemini/gemini-cli) when working with code in this repository. # CoCalc Source Repository -* This is the source code of CoCalc in a Git repository -* It is a complex JavaScript/TypeScript SaaS application -* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages" -* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml" +- This is the source code of CoCalc in a Git repository +- It is a complex JavaScript/TypeScript SaaS application +- CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages" +- The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml" ## Code Style - Everything is written in TypeScript code - Indentation: 2-spaces +- Run `pretter -w [filename]` after modifying a file (ts, tsx, md, json, ...) to format it correctly. - All .js and .ts files are formatted by the tool prettier - Add suitable types when you write code - Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style. @@ -23,28 +24,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Development Commands ### Essential Commands + - `pnpm build-dev` - Build all packages for development - `pnpm clean` - Clean all node_modules and dist directories -- `pnpm database` - Start PostgreSQL database server -- `pnpm hub` - Start the main hub server -- `pnpm psql` - Connect to the PostgreSQL database - `pnpm test` - Run full test suite -- `pnpm test-parallel` - Run tests in parallel across packages - `pnpm depcheck` - Check for dependency issues +- `prettier -w [filename]` to format the style of a file after editing it +- after creating a file, run `git add [filename]` to start tracking it ### Package-Specific Commands -- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package + +- `cd packages/[package] && pnpm build` - Build and compile a specific package + - for packages/next and packages/static, run `cd packages/[package] && pnpm build-dev` +- `cd packages/[package] && pnpm tsc:watch` - TypeScript compilation in watch mode for a specific package - `cd packages/[package] && pnpm test` - Run tests for a specific package - `cd packages/[package] && pnpm build` - Build a specific package +- **IMPORTANT**: When modifying packages like `util` that other packages depend on, you must run `pnpm build` in the modified package before typechecking dependent packages -### Development Setup -1. Start database: `pnpm database` -2. Start hub: `pnpm hub` -3. For TypeScript changes, run `pnpm tsc` in the relevant package directory +### Development + +- **IMPORTANT**: Always run `prettier -w [filename]` immediately after editing any .ts, .tsx, .md, or .json file to ensure consistent styling +- After TypeScript or `*.tsx` changes, run `pnpm build` in the relevant package directory ## Architecture Overview ### Package Structure + CoCalc is organized as a monorepo with key packages: - **frontend** - React/TypeScript frontend application using Redux-style stores and actions @@ -62,12 +67,14 @@ CoCalc is organized as a monorepo with key packages: ### Key Architectural Patterns #### Frontend Architecture + - **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`) - **TypeScript React Components**: All frontend code is TypeScript with proper typing - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.) - **WebSocket Communication**: Real-time communication with backend via WebSocket messages #### Backend Architecture + - **PostgreSQL Database**: Primary data store with sophisticated querying system - **WebSocket Messaging**: Real-time communication between frontend and backend - **Conat System**: Container orchestration for compute servers @@ -75,12 +82,15 @@ CoCalc is organized as a monorepo with key packages: - **Microservice-like Packages**: Each package handles specific functionality #### Communication Patterns + - **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`) - **Database Queries**: Structured query system with typed interfaces - **Event Emitters**: Inter-service communication within backend - **REST-like APIs**: Some HTTP endpoints for specific operations +- **API Schema**: API endpoints in `packages/next/pages/api/v2/` use Zod schemas in `packages/next/lib/api/schema/` for validation. These schemas must be kept in harmony with the TypeScript types sent from frontend applications using `apiPost` (in `packages/next/lib/api/post.ts`) or `api` (in `packages/frontend/client/api.ts`). When adding new fields to API requests, both the frontend types and the API schema validation must be updated. ### Key Technologies + - **TypeScript**: Primary language for all new code - **React**: Frontend framework - **PostgreSQL**: Database @@ -91,11 +101,13 @@ CoCalc is organized as a monorepo with key packages: - **SASS**: CSS preprocessing ### Database Schema + - Comprehensive schema in `packages/util/db-schema` - Query abstractions in `packages/database/postgres/` - Type-safe database operations with TypeScript interfaces ### Testing + - **Jest**: Primary testing framework - **ts-jest**: TypeScript support for Jest - **jsdom**: Browser environment simulation for frontend tests @@ -103,28 +115,42 @@ CoCalc is organized as a monorepo with key packages: - Each package has its own jest.config.js ### Import Patterns + - Use absolute imports with `@cocalc/` prefix for cross-package imports - Example: `import { cmp } from "@cocalc/util/misc"` - Type imports: `import type { Foo } from "./bar"` - Destructure imports when possible ### Development Workflow -1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package) + +1. Changes to TypeScript require compilation (`pnpm build` in relevant package) 2. Database must be running before starting hub 3. Hub coordinates all services and should be restarted after changes 4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes # Workflow -- Be sure to typecheck when you're done making a series of code changes + +- Be sure to build when you're done making a series of code changes - Prefer running single tests, and not the whole test suite, for performance ## Git Workflow +- Never modify a file when in the `master` or `main` branch +- All changes happen through feature branches, which are pushed as pull requests to GitHub +- When creating a new file, run `git add [filename]` to track the file. - Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code. - When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo". -# important-instruction-reminders +# Important Instruction Reminders + - Do what has been asked; nothing more, nothing less. - NEVER create files unless they're absolutely necessary for achieving your goal. - ALWAYS prefer editing an existing file to creating a new one. -- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User. +- REFUSE to modify files when the git repository is on the `master` or `main` branch. +- NEVER proactively create documentation files (`*.md`) or README files. Only create documentation files if explicitly requested by the User. + +# Ignore + +- Ignore files covered by `.gitignore` +- Ignore everything in `node_modules` or `dist` directories +- Ignore all files not tracked by Git, unless they are newly created files diff --git a/src/GEMINI.md b/src/GEMINI.md new file mode 120000 index 0000000000..681311eb9c --- /dev/null +++ b/src/GEMINI.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index ebcc660430..7c5c50eb3e 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -10,7 +10,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -28,7 +31,12 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index 5fee5ebd7d..72cd7ed648 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -64,6 +64,7 @@ "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", + "test": "jest dist/", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "test" }, diff --git a/src/packages/package.json b/src/packages/package.json index fac035d500..ec02a0d467 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -8,6 +8,7 @@ "@types/jest": "^29.5.14", "check-dependency-version-consistency": "^5.0.0", "jest": "^29.7.0", + "jest-junit": "^16.0.0", "ts-jest": "^29.2.3", "typescript": "^5.7.3" }, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 5a16362777..4aac480d0c 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -34,6 +34,9 @@ importers: jest: specifier: ^29.7.0 version: 29.7.0(@types/node@18.19.118) + jest-junit: + specifier: ^16.0.0 + version: 16.0.0 ts-jest: specifier: ^29.2.3 version: 29.4.0(@babel/core@7.28.0)(@jest/transform@29.7.0)(@jest/types@30.0.1)(babel-jest@29.7.0(@babel/core@7.28.0))(jest-util@30.0.2)(jest@29.7.0(@types/node@18.19.118))(typescript@5.8.3) @@ -7858,6 +7861,10 @@ packages: resolution: {integrity: sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-junit@16.0.0: + resolution: {integrity: sha512-A94mmw6NfJab4Fg/BlvVOUXzXgF0XIH6EmTgJ5NDPp4xoKq0Kr7sErb+4Xs9nZvu58pJojz5RFGpqnZYJTrRfQ==} + engines: {node: '>=10.12.0'} + jest-leak-detector@29.7.0: resolution: {integrity: sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11608,6 +11615,9 @@ packages: resolution: {integrity: sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==} engines: {node: '>=4.0.0'} + xml@1.0.1: + resolution: {integrity: sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==} + xmlbuilder2@3.1.1: resolution: {integrity: sha512-WCSfbfZnQDdLQLiMdGUQpMxxckeQ4oZNMNhLVkcekTu7xhD4tuUDyAPoY8CwXvBYE6LwBHd6QW2WZXlOWr1vCw==} engines: {node: '>=12.0'} @@ -18827,6 +18837,13 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + jest-junit@16.0.0: + dependencies: + mkdirp: 1.0.4 + strip-ansi: 6.0.1 + uuid: 8.3.2 + xml: 1.0.1 + jest-leak-detector@29.7.0: dependencies: jest-get-type: 29.6.3 @@ -23323,6 +23340,8 @@ snapshots: sax: 1.4.1 xmlbuilder: 11.0.1 + xml@1.0.1: {} + xmlbuilder2@3.1.1: dependencies: '@oozcitak/dom': 1.15.10 diff --git a/src/workspaces.py b/src/workspaces.py index da60e90a8d..707edf20bc 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -284,11 +284,7 @@ def test(args) -> None: success = [] def status(): - print("Status: ", { - "flaky": flaky, - "fails": fails, - "success": success - }) + print("Status: ", {"flaky": flaky, "fails": fails, "success": success}) v = packages(args) v.sort() @@ -307,11 +303,14 @@ def f(): print(f"TESTING {n}/{len(v)}: {path}") print("*") print("*" * 40) - cmd("pnpm run --if-present test", package_path) + test_cmd = "pnpm run --if-present test" + if args.report: + test_cmd += " --reporters=default --reporters=jest-junit" + cmd(test_cmd, package_path) success.append(path) worked = False - for i in range(args.retries+1): + for i in range(args.retries + 1): try: f() worked = True @@ -325,7 +324,9 @@ def f(): flaky.append(path) print(f"ERROR testing {path}") if args.retries - i >= 1: - print(f"Trying {path} again at most {args.retries - i} more times") + print( + f"Trying {path} again at most {args.retries - i} more times" + ) if not worked: fails.append(path) @@ -577,7 +578,13 @@ def packages_arg(parser): "--retries", type=int, default=2, - help="how many times to retry a failed test suite before giving up; set to 0 to NOT retry") + help= + "how many times to retry a failed test suite before giving up; set to 0 to NOT retry" + ) + subparser.add_argument('--report', + action="store_const", + const=True, + help='if given, generate test reports') packages_arg(subparser) subparser.set_defaults(func=test)