cocalc-api/mcp: jupyter exec and more test fixes

Author: haraldschilly

src/cocalc.code-workspace

@@ -3,6 +3,9 @@
     {
       "name": "cocalc",
       "path": "."
+    },
+    {
+      "path": "../.github"
     }
   ],
   "settings": {

src/packages/server/api/project-bridge.ts

@@ -3,6 +3,7 @@
 import { projectSubject } from "@cocalc/conat/names";
 import { conat } from "@cocalc/backend/conat";
 import { type Client as ConatClient } from "@cocalc/conat/core/client";
+import { getProject } from "@cocalc/server/projects/control";
 const DEFAULT_TIMEOUT = 15000;
 
 let client: ConatClient | null = null;
@@ -51,6 +52,12 @@ async function callProject({
     service: "api",
   });
   try {
+    // Ensure the project is running before making the API call
+    const project = getProject(project_id);
+    if (project) {
+      await project.start();
+    }
+
     // For system.test(), inject project_id into args[0] if not already present
     let finalArgs = args;
     if (name === "system.test" && (!args || args.length === 0)) {

src/python/cocalc-api/src/cocalc_api/mcp/DEVELOPMENT.md

@@ -123,6 +123,7 @@ The MCP server exposes high-level metadata that clients can query to understand
 **Protocol Version** - MCP 2025-06-18
 
 Clients can retrieve this metadata using the MCP `initialize` call, which returns:
+
 - `serverInfo.name` - "cocalc-api"
 - `serverInfo.version` - Version from package metadata
 - `instructions` - High-level guide (see above)
@@ -192,6 +193,41 @@ Execute arbitrary shell commands in the CoCalc project.
 {"command": "echo 'Hello'"}
 {"command": "python", "args": ["script.py", "--verbose"]}
 {"command": "for i in {1..3}; do echo $i; done", "bash": true}
+{"command": "jupyter kernelspec list"}
+{"command": "python3", "args": ["-m", "pip", "install", "--user", "ipykernel"]}
+```
+
+#### `jupyter_execute` - Execute Code in Jupyter Kernels
+
+Execute code using Jupyter kernels with rich output, preserved state, and support for multiple languages.
+
+**Parameters:**
+
+- `input` (string, required): Code to execute
+- `kernel` (string, optional): Kernel name (default: "python3")
+  - Common kernels: `python3`, `ir` (R), `julia-1.9`
+  - Use `exec` tool with `jupyter kernelspec list` to discover available kernels
+- `history` (list, optional): Previous code inputs to establish context
+  - Executed without capturing output, allows setting up variables and imports
+
+**Returns:** Formatted execution output (text, plots, dataframes, images, errors, etc.)
+
+**Examples:**
+
+```json
+{"input": "2 + 2", "kernel": "python3"}
+{"input": "import pandas as pd\ndf = pd.DataFrame({'a': [1,2,3]})\ndf", "kernel": "python3"}
+{"input": "import matplotlib.pyplot as plt\nplt.plot([1,2,3])\nplt.show()", "kernel": "python3"}
+{"input": "summary(cars)", "kernel": "ir"}
+```
+
+**Setup Instructions:**
+
+Before using Jupyter kernels, set them up with the `exec` tool:
+
+```json
+{"command": "python3", "args": ["-m", "pip", "install", "--user", "ipykernel"], "timeout": 300}
+{"command": "python3", "args": ["-m", "ipykernel", "install", "--user", "--name=python3", "--display-name=Python 3"], "timeout": 120}
 ```
 
 ### Resources

src/python/cocalc-api/src/cocalc_api/mcp/README.md

@@ -55,6 +55,7 @@ claude mcp add-json cocalc '{
 ```
 
 **Important:**
+
 - Replace `/path/to/cocalc/src/python/cocalc-api` with the absolute path to your cocalc-api directory.
 - Replace `http://localhost:5000` with your CoCalc instance URL (defaults to `https://cocalc.com` if not set).
 
@@ -67,7 +68,12 @@ Add to `~/.config/Claude/claude_desktop_config.json`:
   "mcpServers": {
     "cocalc": {
       "command": "uv",
-      "args": ["--directory", "/path/to/cocalc/src/python/cocalc-api", "run", "cocalc-mcp-server"],
+      "args": [
+        "--directory",
+        "/path/to/cocalc/src/python/cocalc-api",
+        "run",
+        "cocalc-mcp-server"
+      ],
       "env": {
         "COCALC_API_KEY": "sk-your-api-key-here",
         "COCALC_PROJECT_ID": "[UUID]",
@@ -79,6 +85,7 @@ Add to `~/.config/Claude/claude_desktop_config.json`:
 ```
 
 **Important:**
+
 - Replace `/path/to/cocalc/src/python/cocalc-api` with the absolute path to your cocalc-api directory.
 - Replace `http://localhost:5000` with your CoCalc instance URL (defaults to `https://cocalc.com` if not set).
 
@@ -88,13 +95,12 @@ To automatically allow all CoCalc MCP tools without prompts, add this to `.claud
 
 ```json
 {
-  "allowedTools": [
-    "mcp__cocalc__*"
-  ]
+  "allowedTools": ["mcp__cocalc__*"]
 }
 ```
 
 This wildcard pattern (`mcp__cocalc__*`) automatically allows:
+
 - `mcp__cocalc__exec` - Execute shell commands
 - `mcp__cocalc__project_files` - Browse project files
 - Any future tools added to the MCP server
@@ -104,12 +110,20 @@ This wildcard pattern (`mcp__cocalc__*`) automatically allows:
 ### Tools
 
 - **`exec`** - Execute shell commands in the project
+
   ```
   Tool: exec
   Params: command (required), args, bash, timeout, cwd
   Returns: {stdout, stderr, exit_code}
   ```
 
+- **`jupyter_execute`** - Execute code using Jupyter kernels
+  ```
+  Tool: jupyter_execute
+  Params: input (required), kernel (default: "python3"), history
+  Returns: Formatted execution output (text, plots, errors, etc.)
+  ```
+
 ### Resources
 
 - **`project-files`** - Browse project files with filtering and pagination
@@ -121,6 +135,7 @@ This wildcard pattern (`mcp__cocalc__*`) automatically allows:
 ## Documentation
 
 See [DEVELOPMENT.md](./DEVELOPMENT.md) for:
+
 - Architecture and design principles
 - Detailed API specifications
 - Configuration options

src/python/cocalc-api/src/cocalc_api/mcp/mcp_debug.py

@@ -19,7 +19,6 @@
 """
 
 import asyncio
-import json
 import os
 import sys
 

src/python/cocalc-api/src/cocalc_api/mcp/mcp_server.py

@@ -6,6 +6,8 @@
 AVAILABLE TOOLS (actions you can perform):
 - exec: Run shell commands, scripts, and programs in the project
   Use for: running code, data processing, build/test commands, git operations, etc.
+- jupyter_execute: Execute code using Jupyter kernels (Python, R, Julia, etc.)
+  Use for: interactive code execution, data analysis, visualization, scientific computing
 
 AVAILABLE RESOURCES (information you can read):
 - project-files: Browse the project file structure
@@ -33,13 +35,40 @@
 
 import os
 import sys
+import time
 from typing import Optional
 
 from mcp.server.fastmcp import FastMCP
 
 from cocalc_api import Project, Hub
 
 
+def _retry_with_backoff(func, max_retries: int = 3, retry_delay: int = 2):
+    """
+    Retry a function with exponential backoff for transient failures.
+
+    Used during server initialization for operations that may timeout on cold starts.
+    """
+    for attempt in range(max_retries):
+        try:
+            return func()
+        except Exception as e:
+            error_msg = str(e).lower()
+            is_retryable = any(
+                keyword in error_msg
+                for keyword in ["timeout", "closed", "connection", "reset", "broken"]
+            )
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Initialization attempt {attempt + 1} failed ({error_msg[:50]}...), "
+                    f"retrying in {retry_delay}s...",
+                    file=sys.stderr,
+                )
+                time.sleep(retry_delay)
+            else:
+                raise
+
+
 def get_config() -> tuple[str, str, Optional[str]]:
     """
     Get and validate MCP server configuration.
@@ -106,6 +135,7 @@ def check_api_key_scope(api_key: str, host: str) -> dict[str, str]:
 This server gives you direct access to a CoCalc project or account environment through the Model Context Protocol (MCP).
 
 WHAT YOU CAN DO:
+- Execute code using Jupyter kernels (Python, R, Julia, etc.) with rich output and visualization support
 - Execute arbitrary shell commands in a Linux environment with Python, Node.js, R, Julia, and 100+ tools
 - Browse and explore project files to understand structure and contents
 - Run code, scripts, and build/test commands
@@ -114,11 +144,14 @@ def check_api_key_scope(api_key: str, host: str) -> dict[str, str]:
 
 HOW TO USE:
 1. Start by exploring the project structure using the project-files resource
-2. Use the exec tool to run commands, scripts, or programs
-3. Combine multiple commands to accomplish complex workflows
+2. Use jupyter_execute for interactive code execution with rich output (plots, tables, etc.)
+3. Use exec tool to run shell commands, scripts, or programs
+4. Combine multiple commands to accomplish complex workflows
 
 EXAMPLES:
-- Execute Python: exec with command="python3 script.py --verbose"
+- Execute Python interactively: jupyter_execute with input="import pandas as pd; df = pd.read_csv('data.csv'); df.describe()"
+- Data visualization: jupyter_execute with input="import matplotlib.pyplot as plt; plt.plot([1,2,3]); plt.show()"
+- Execute shell command: exec with command="python3 script.py --verbose"
 - List files: use project-files resource or exec with command="ls -la"
 - Run tests: exec with command="pytest tests/" bash=true
 - Git operations: exec with command="git log --oneline" in your repository
@@ -157,22 +190,12 @@ def _initialize_config() -> None:
         try:
             _api_key_scope = check_api_key_scope(_api_key, _host)
         except RuntimeError as check_error:
-            # If it's a project-scoped key error, try the project API to discover the project_id
+            # If it's a project-scoped key error, use a placeholder project_id
+            # Project-scoped keys have the project_id embedded in the key itself
             if "project-scoped" in str(check_error):
-                try:
-                    # Try with empty project_id - project-scoped keys will use their own
-                    project = Project(api_key=_api_key, project_id="", host=_host)
-                    result = project.system.ping()
-                    # Check if the response includes project_id (it shouldn't from ping, but try anyway)
-                    if isinstance(result, dict) and "project_id" in result:
-                        _api_key_scope = {"project_id": result["project_id"]}
-                    else:
-                        # If we still don't have it, this is an error
-                        raise RuntimeError("Could not determine project_id from project-scoped API key. "
-                                           "Please restart with COCALC_PROJECT_ID environment variable.")
-                except Exception as project_error:
-                    raise RuntimeError(f"Project-scoped API key detected but could not determine project_id. "
-                                       f"Error: {project_error}") from project_error
+                # Use empty string as project_id - the Project client will extract it from the API key
+                _api_key_scope = {"project_id": ""}
+                print("✓ Connected with project-scoped API key", file=sys.stderr)
             else:
                 raise
 
@@ -181,12 +204,15 @@ def _initialize_config() -> None:
             print(f"✓ Connected with account-scoped API key (account: {account_id})", file=sys.stderr)
         elif "project_id" in _api_key_scope:
             project_id = _api_key_scope["project_id"]
-            if not project_id:
-                raise RuntimeError("Project ID not found for project-scoped API key")
-            print(f"✓ Connected with project-scoped API key (project: {project_id})", file=sys.stderr)
-            # For project-scoped keys, eagerly create the project client
-            client = Project(api_key=_api_key, project_id=project_id, host=_host)
-            _project_clients[project_id] = client
+            # For project-scoped keys with empty/None project_id, the Project client will extract it from the API key
+            if project_id:
+                print(f"✓ Connected with project-scoped API key (project: {project_id})", file=sys.stderr)
+                # For project-scoped keys, eagerly create the project client
+                client = Project(api_key=_api_key, project_id=project_id, host=_host)
+                _project_clients[project_id] = client
+            else:
+                # Project-scoped key with empty project_id - will be discovered on first use
+                print("✓ Connected with project-scoped API key (project ID will be discovered on first use)", file=sys.stderr)
         else:
             # If we got here with no project_id but it might be project-scoped, check if COCALC_PROJECT_ID was provided
             if project_id_config:
@@ -232,19 +258,24 @@ def get_project_client(project_id: Optional[str] = None) -> Project:
             raise RuntimeError("Account-scoped API key requires an explicit project_id argument. "
                                "No project_id provided to get_project_client().")
 
-    if not project_id:
-        raise RuntimeError("Project ID cannot be empty")
+    # For project-scoped keys with None/empty project_id, the Project client will extract it from the API key
+    # For account-scoped keys, project_id must be non-empty
+    if not project_id and _api_key_scope and "account_id" in _api_key_scope:
+        raise RuntimeError("Account-scoped API key requires a non-empty project_id")
+
+    # Use a cache key that handles None/empty project_id for project-scoped keys
+    cache_key = project_id if project_id else "_default_project"
 
     # Return cached client if available
-    if project_id in _project_clients:
-        return _project_clients[project_id]
+    if cache_key in _project_clients:
+        return _project_clients[cache_key]
 
     # Create new project client
     # At this point, _api_key and _host are guaranteed to be non-None (set in _initialize_config)
     assert _api_key is not None
     assert _host is not None
     client = Project(api_key=_api_key, project_id=project_id, host=_host)
-    _project_clients[project_id] = client
+    _project_clients[cache_key] = client
     return client
 
 

src/python/cocalc-api/src/cocalc_api/mcp/tools/__init__.py

@@ -5,6 +5,7 @@
 
 Available Tools:
 - exec: Execute shell commands, scripts, and programs in the project environment
+- jupyter_execute: Execute code using Jupyter kernels with rich output and interactive state
 
 See mcp_server.py for overview of all available tools and resources, and guidance
 on when to use each one.
@@ -14,5 +15,7 @@
 def register_tools(mcp) -> None:
     """Register all tools with the given FastMCP instance."""
     from .exec import register_exec_tool
+    from .jupyter import register_jupyter_tool
 
     register_exec_tool(mcp)
+    register_jupyter_tool(mcp)

src/python/cocalc-api/src/cocalc_api/mcp/tools/exec.py

@@ -37,6 +37,14 @@ async def exec(
         The command executes in the project's Linux shell environment with access to the
         project's file system and all installed packages/tools.
 
+        Common use cases:
+        - List available Jupyter kernels: exec(command="jupyter kernelspec list")
+        - Install ipykernel: exec(command="python3", args=["-m", "pip", "install", "--user", "ipykernel"])
+        - Register Python kernel: exec(command="python3", args=["-m", "ipykernel", "install", "--user", "--name=python3", "--display-name=Python 3"])
+        - Install packages: exec(command="pip", args=["install", "pandas"], bash=False)
+        - Run scripts: exec(command="python", args=["script.py"])
+        - Execute complex pipelines: exec(command="cat data.txt | grep pattern | wc -l", bash=True)
+
         Args:
             command: The command to execute (e.g., 'ls -la', 'python script.py', 'echo 2 + 3 | bc')
             args: Optional list of arguments to pass to the command