Add scripts and documentation for tests and common development flows

.github/workflows/ci.yml

@@ -75,37 +75,6 @@ jobs:
         run: |
           RUSTDOCFLAGS='--deny warnings' cargo doc --no-deps
 
-  frontend_tests:
-    name: frontend tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-
-      - name: Setup NodeJS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 24
-          cache: "pnpm"
-
-      - name: Install Rust toolchain from file
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          # Don't override flags in cargo config files.
-          rustflags: ""
-
-      - name: Build frontend
-        run: |
-          pnpm install
-          pnpm --filter ./packages/frontend run build -- --mode development
-
-      - name: Run frontend tests
-        run: |
-          pnpm --filter ./packages/frontend run test:no-backend
-
   ui_components_tests:
     name: ui-components tests
     runs-on: ubuntu-latest
@@ -131,37 +100,6 @@ jobs:
       - name: Run ui-components tests
         run: pnpm --filter ./packages/ui-components run test
 
-  backend_dev_setup:
-    name: backend dev setup
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Install Rust toolchain from file
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-
-      - name: Setup wasm-pack
-        uses: jetli/wasm-pack-action@v0.4.0
-        with:
-          version: 'v0.13.1'
-
-      - name: Run dev setup
-        run: |
-            docker run --name catcolab-postgres -e POSTGRES_USER=postgres-user -e POSTGRES_PASSWORD=password -e POSTGRES_DB=catcolab -p 5432:5432 -d postgres:15
-            cd packages/notebook-types
-            pnpm run build:node
-            cd ../automerge-doc-server
-            pnpm install
-            cd ../backend
-            cp .env.development .env && cp .env.development ../migrator/.env
-            # due to how we are detecting deploy vs dev we need to clear this env variable
-            unset INVOCATION_ID
-            cargo run -p migrator apply
-
   npm_checks:
     name: npm checks
     runs-on: ubuntu-latest
@@ -208,3 +146,26 @@ jobs:
     - name: Build the NixOS system for catcolab-next
       run: nix build .#nixosConfigurations.catcolab-next.config.system.build.toplevel
 
+  frontend_tests_with_backend:
+    name: frontend tests with backend
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install Nix
+      uses: cachix/install-nix-action@v25
+      with:
+        install_url: https://releases.nixos.org/nix/nix-2.29.2/install
+
+    - name: Configure Cachix
+      uses: cachix/cachix-action@v14
+      with:
+        name: catcolab-jmoggr
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+    - name: Run frontend tests with VM
+      run: |
+        nix build .#checks.x86_64-linux.frontendTests -L --no-sandbox
+

dev-docs/README.md

@@ -0,0 +1,40 @@
+# Developer Documentation
+
+This directory contains documentation for CatColab developers working on testing, deployment, and local development workflows.
+
+## Guides
+
+### [Testing Guide](docs/testing-guide.md)
+Learn how to run integration tests, test deployments, and debug test failures with a progressive isolation strategy.
+
+### [Development Workflows](docs/workflows.md)
+Example workflows demonstrating how to use `cc-utils` script for common development tasks.
+
+## Reference
+
+### [cc-utils vm](docs/references/cc-utils-vm.md)
+Reference for `cc-utils vm` commands used to manage local NixOS VMs.
+
+### [cc-utils db](docs/references/cc-utils-db.md)
+Reference for `cc-utils db` commands used to manage databases across environments.
+
+## Quick Start
+
+**Preview remote branch:**
+```bash
+cc-utils vm start my-feature-branch --db cached
+```
+
+**Test a deployment:**
+```bash
+cc-utils vm test-deploy
+```
+
+**Work with databases:**
+```bash
+# Dump staging database
+cc-utils db dump --from staging
+
+# Load most recent staging dump into local db
+cc-utils db load --from staging
+```

dev-docs/docs/references/cc-utils-db.md

@@ -0,0 +1,52 @@
+# cc-utils DB Commands Reference
+
+The `cc-utils db` commands provide tools for managing databases across environments (local, staging, production, vm).
+
+## Available Commands
+
+### connect
+
+```bash
+cc-utils db connect <target>
+```
+
+Opens a `psql` connection to the specified database.
+
+**Targets:** `local`, `staging`, `production`, `vm`
+
+### dump
+
+```bash
+cc-utils db dump [--from <target>] [--to <filename>] [--quiet]
+```
+
+Creates a SQL dump file. Defaults to `local` and generates a timestamped filename.
+
+**Options:**
+- `--from`: Source database (local, staging, production, vm)
+- `--to`: Output filename (stored in `./dumps/`)
+- `--quiet`: Only output the dump file path
+
+### load
+
+```bash
+cc-utils db load --from <file|source> [--to <target>] [--skip-migrations]
+```
+
+Loads a SQL dump into a database and runs migrations by default.
+
+**Options:**
+- `--from`: Dump file path or source name (local, staging, production) for most recent dump
+- `--to`: Target database (local or vm)
+- `--skip-migrations`: Skip running migrations after loading
+
+### reset
+
+```bash
+cc-utils db reset [--skip-migrations]
+```
+
+Drops and recreates the local database. Only supports `local` target.
+
+**Options:**
+- `--skip-migrations`: Skip running migrations after reset

dev-docs/docs/references/cc-utils-vm.md

@@ -0,0 +1,82 @@
+# cc-utils VM Commands Reference
+
+The `cc-utils vm` commands provide tools for managing local NixOS VMs for testing and development.
+
+## Available Commands
+
+### start
+
+```bash
+cc-utils vm start [branch] [--db skip|cached|<file>]
+```
+
+Starts a VM, optionally from a specific GitHub branch.
+
+**Arguments:**
+- `branch` (optional) - GitHub branch name to build VM from. If omitted, uses local nix configuration.
+
+**Database options:**
+- No `--db` flag (default) - Load fresh staging database dump
+- `--db skip` - Don't load any database
+- `--db cached` - Use most recent staging dump in dumps/, or create new if missing
+- `--db <file>` - Load from specific dump file path
+
+**Examples:**
+```bash
+# Start local VM with fresh staging DB
+cc-utils vm start
+
+# Start VM from branch with cached DB
+cc-utils vm start my-feature-branch --db cached
+
+# Start without loading database
+cc-utils vm start --db skip
+
+# Start with specific dump file
+cc-utils vm start --db ./dumps/my_dump.sql
+```
+
+### stop
+
+```bash
+cc-utils vm stop
+```
+
+Gracefully shuts down the running VM.
+
+### status
+
+```bash
+cc-utils vm status
+```
+
+Shows whether the VM is running and displays the log file location.
+
+### logs
+
+```bash
+cc-utils vm logs [--follow]
+```
+
+Displays VM logs. Use `--follow` (or `-f`) to tail the logs in real-time.
+
+### connect
+
+```bash
+cc-utils vm connect
+```
+
+Opens an SSH connection to the running VM. Useful for inspecting the VM state or running commands manually.
+
+### test-deploy
+
+```bash
+cc-utils vm test-deploy
+```
+
+Comprehensive test workflow that:
+1. Builds a VM from the current staging branch on GitHub
+2. Starts the VM
+3. Copies the staging database to the VM
+4. Deploys your current local branch to the running VM
+5. Runs the frontend tests