cocalc-api: update, tweaks for jupyter testing, detecting older api key

Author: haraldschilly

src/packages/server/api/manage.ts

@@ -295,8 +295,9 @@ export async function getAccountWithApiKey(
     return;
   }
 
-  // Check for legacy account api key (format: sk_*)
-  if (secret.startsWith("sk_")) {
+  // Check for legacy account api key (format historically documented as sk-*, but
+  // some deployments used sk_*, so accept both to avoid breaking existing keys)
+  if (secret.startsWith(API_KEY_PREFIX) || secret.startsWith("sk_")) {
     const { rows } = await pool.query(
       "SELECT account_id FROM accounts WHERE api_key = $1::TEXT",
       [secret],

src/python/cocalc-api/src/cocalc_api/hub.py

@@ -10,8 +10,8 @@ class Hub:
     def __init__(self, api_key: str, host: str = "https://cocalc.com"):
         self.api_key = api_key
         self.host = host
-        # Use longer timeout for API calls (30 seconds instead of default 5)
-        self.client = httpx.Client(auth=(api_key, ""), headers={"Content-Type": "application/json"}, timeout=30.0)
+        # Use longer timeout for API calls (90 seconds instead of default 5) to handle slow operations like Jupyter
+        self.client = httpx.Client(auth=(api_key, ""), headers={"Content-Type": "application/json"}, timeout=90.0)
 
     def call(self, name: str, arguments: list[Any], timeout: Optional[int] = None) -> Any:
         """
@@ -326,14 +326,15 @@ def kernels(self, project_id: Optional[str] = None) -> list[dict[str, Any]]:
         """
         ...
 
-    @api_method("jupyter.execute")
+    @api_method("jupyter.execute", timeout_seconds=True)
     def execute(
         self,
         input: str,
         kernel: str,
         history: Optional[list[str]] = None,
         project_id: Optional[str] = None,
         path: Optional[str] = None,
+        timeout: Optional[int] = 30,
     ) -> dict[str, Any]:  # type: ignore[empty-body]
         """
         Execute code using a Jupyter kernel.
@@ -344,6 +345,7 @@ def execute(
             history (Optional[list[str]]): Array of previous inputs (they get evaluated every time, but without output being captured).
             project_id (Optional[str]): Project in which to run the code -- if not given, global anonymous project is used, if available.
             path (Optional[str]): File path context for execution.
+            timeout (Optional[int]): Timeout in SECONDS for the execute call (defaults to 30 seconds).
 
         Returns:
             dict[str, Any]: JSON response containing execution results.

src/python/cocalc-api/src/cocalc_api/project.py

@@ -124,13 +124,14 @@ def exec(
         """
         ...
 
-    @api_method("system.jupyterExecute")
+    @api_method("system.jupyterExecute", timeout_seconds=True)
     def jupyter_execute(
         self,
         input: str,
         kernel: str,
         history: Optional[list[str]] = None,
         path: Optional[str] = None,
+        timeout: Optional[int] = 30,
     ) -> list[dict[str, Any]]:  # type: ignore[empty-body]
         """
         Execute code using a Jupyter kernel.
@@ -140,6 +141,7 @@ def jupyter_execute(
             kernel (str): Name of kernel to use. Get options using hub.jupyter.kernels().
             history (Optional[list[str]]): Array of previous inputs (they get evaluated every time, but without output being captured).
             path (Optional[str]): File path context for execution.
+            timeout (Optional[int]): Timeout in SECONDS for the execute call (defaults to 30 seconds).
 
         Returns:
             list[dict[str, Any]]: List of output items. Each output item contains

src/python/cocalc-api/tests/conftest.py

@@ -1,6 +1,7 @@
 """
 Pytest configuration and fixtures for cocalc-api tests.
 """
+import json
 import os
 import time
 import uuid
@@ -93,16 +94,20 @@ def temporary_project(hub, resource_tracker, request):
         # Wait for project to be ready (can take 10-15 seconds)
         from cocalc_api import Project
 
+        test_project = Project(project_id=project_id, api_key=hub.api_key, host=hub.host)
         for attempt in range(10):
             time.sleep(5)  # Wait 5 seconds before checking
             try:
                 # Try to ping the project to see if it's ready
-                test_project = Project(project_id=project_id, api_key=hub.api_key, host=hub.host)
                 test_project.system.ping()  # If this succeeds, project is ready
                 break
             except Exception:
                 if attempt == 9:  # Last attempt
                     print(f"Warning: Project {project_id} did not become ready within 50 seconds")
+        else:
+            print(f"Warning: Project {project_id} may not be ready yet")
+
+        ensure_python3_kernel(test_project)
 
     except Exception as e:
         print(f"Warning: Failed to start project {project_id}: {e}")
@@ -120,6 +125,53 @@ def project_client(temporary_project, api_key, cocalc_host):
     return Project(project_id=temporary_project['project_id'], api_key=api_key, host=cocalc_host)
 
 
+def ensure_python3_kernel(project_client: Project):
+    """
+    Ensure the default python3 Jupyter kernel is installed in the project.
+
+    If not available, install ipykernel and register the kernelspec.
+    """
+
+    def has_python_kernel() -> bool:
+        try:
+            result = project_client.system.exec(
+                command="python3",
+                args=["-m", "jupyter", "kernelspec", "list", "--json"],
+                timeout=60,
+            )
+            data = json.loads(result["stdout"])
+            kernelspecs = data.get("kernelspecs", {})
+            return "python3" in kernelspecs
+        except Exception as err:
+            print(f"Warning: Failed to list kernelspecs: {err}")
+            return False
+
+    if has_python_kernel():
+        return
+
+    print("Installing python3 kernelspec in project...")
+    project_client.system.exec(
+        command="python3",
+        args=["-m", "pip", "install", "--user", "ipykernel"],
+        timeout=300,
+    )
+    project_client.system.exec(
+        command="python3",
+        args=[
+            "-m",
+            "ipykernel",
+            "install",
+            "--user",
+            "--name=python3",
+            "--display-name=Python 3",
+        ],
+        timeout=120,
+    )
+
+    if not has_python_kernel():
+        raise RuntimeError("Failed to ensure python3 kernelspec is installed in project")
+
+
 # ============================================================================
 # Database Cleanup Infrastructure
 # ============================================================================
@@ -218,7 +270,8 @@ def db_pool(check_cleanup_config):
     if not cleanup_enabled:
         print("\n⚠ Database cleanup DISABLED via COCALC_TESTS_CLEANUP=false")
         print("   Test resources will remain in the database.")
-        return None
+        yield None
+        return
 
     # Get connection parameters with defaults
     pguser = os.environ.get("PGUSER", "smc")