fix(installer): Prioritize system site packages over server-libs (#38)

* fix(installer): prioritize user-installed packages over server-libs

Change sys.path.insert(0, ...) to sys.path.append(...) for server
site-packages so user-installed packages in system site-packages
take precedence over bundled server-libs packages.

Previously, server-libs were inserted at the front of sys.path,
causing bundled versions to override user-installed versions
(e.g., streamlit).

* fix(installer): prioritize user-installed packages over server-libs

Change sys.path.insert(0, ...) to sys.path.append(...) for server
site-packages so user-installed packages in system site-packages
take precedence over bundled server-libs packages.

Previously, server-libs were inserted at the front of sys.path,
causing bundled versions to override user-installed versions
(e.g., streamlit).

* fix(installer): ensure system site-packages has priority over server-libs

Add priority parameter to import_package_bundle that uses sys.path.insert(0)
to place packages at the front of sys.path. System site-packages now uses
this priority flag to ensure user-installed packages are found before
bundled server-libs packages.

* test(installer): add tests for import_package_bundle priority param

Add unit tests for VirtualEnvironment.import_package_bundle covering:
- Plain path appending to .pth file
- Conditional import with env var check
- Priority flag using sys.path.insert(0, ...)
- Ordering of bundles matching __main__.py usage

* refactor(installer): improve import_package_bundle and tests

- Add guard to raise ValueError when both condition_env and priority
  are specified (mutually exclusive parameters)
- Update docstring to document the mutual exclusivity constraint
- Refactor tests to use pathlib.Path instead of os.path/os.makedirs
- Add test for mutually exclusive parameter validation

* style(installer): make priority param keyword-only

Add bare * before priority parameter to enforce keyword-only usage,
satisfying Ruff FBT rules for boolean parameters.

Author: mfranczel

installer/__main__.py

@@ -273,7 +273,7 @@ def bootstrap():
     venv.import_package_bundle(
         server_site_packages_path, condition_env="DEEPNOTE_INCLUDE_SERVER_PACKAGES"
     )
-    venv.import_package_bundle(system_site_packages_path)
+    venv.import_package_bundle(system_site_packages_path, priority=True)
     venv.import_package_bundle(kernel_site_package_path)
 
     # Phase 7.2: Symlink notebook to jupyter_server for compatibility with

installer/module/virtual_environment.py

@@ -81,15 +81,29 @@ def start_server(self, command: str, cwd: Optional[str] = None) -> ServerProcess
         return server_proc
 
     def import_package_bundle(
-        self, bundle_site_package_path: str, condition_env: Optional[str] = None
+        self,
+        bundle_site_package_path: str,
+        condition_env: Optional[str] = None,
+        *,
+        priority: bool = False,
     ) -> None:
         """
         Import a package bundle to the virtual environment.
 
         :param bundle_site_package_path: Absolute path to the package bundle.
         :param condition_env: Optional environment variable name. If provided, the bundle
-                         will only be loaded when this env var is set to 'true'.
+                        will only be loaded when this env var is set to 'true'.
+                        Cannot be combined with priority.
+        :param priority: If True, insert at front of sys.path to override other bundles.
+                        Cannot be combined with condition_env.
+        :raises ValueError: If both condition_env and priority are specified.
         """
+        if condition_env and priority:
+            raise ValueError(
+                "condition_env and priority are mutually exclusive; "
+                "specify only one of them"
+            )
+
         pth_file_path = os.path.join(self._site_packages_path, "deepnote.pth")
 
         with open(pth_file_path, "a+", encoding="utf-8") as pth_file:
@@ -105,6 +119,15 @@ def import_package_bundle(
                     "Bundle was conditionally imported to the virtual environment "
                     f"(loads when {condition_env}=true)."
                 )
+            elif priority:
+                # Insert at front of sys.path for higher priority (overrides other bundles)
+                pth_content = (
+                    f"import sys; sys.path.insert(0, '{bundle_site_package_path}')"
+                )
+                pth_file.write(pth_content + "\n")
+                logger.info(
+                    "Bundle was imported with priority to the virtual environment."
+                )
             else:
                 pth_file.write(bundle_site_package_path + "\n")
                 logger.info(

tests/unit/test_virtual_environment.py

@@ -0,0 +1,100 @@
+"""Tests for VirtualEnvironment class."""
+
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from installer.module.virtual_environment import VirtualEnvironment
+
+
+class TestImportPackageBundle:
+    """Tests for import_package_bundle method."""
+
+    @pytest.fixture
+    def venv(self, tmp_path):
+        """Create a VirtualEnvironment with a temporary path."""
+        venv_path = tmp_path / "venv"
+        with patch(
+            "installer.module.virtual_environment.get_current_python_version",
+            return_value="3.11",
+        ):
+            venv = VirtualEnvironment(str(venv_path))
+        # Create the site-packages directory
+        Path(venv.site_packages_path).mkdir(parents=True, exist_ok=True)
+        return venv
+
+    def test_import_package_bundle_plain_path(self, venv):
+        """Test that plain path is appended to .pth file."""
+        bundle_path = "/some/bundle/site-packages"
+
+        venv.import_package_bundle(bundle_path)
+
+        pth_file = Path(venv.site_packages_path) / "deepnote.pth"
+        content = pth_file.read_text()
+
+        assert content == f"{bundle_path}\n"
+
+    def test_import_package_bundle_with_condition_env(self, venv):
+        """Test that conditional import uses sys.path.insert(0, ...)."""
+        bundle_path = "/server/libs/site-packages"
+        condition_env = "DEEPNOTE_INCLUDE_SERVER_PACKAGES"
+
+        venv.import_package_bundle(bundle_path, condition_env=condition_env)
+
+        pth_file = Path(venv.site_packages_path) / "deepnote.pth"
+        content = pth_file.read_text()
+
+        assert "import os, sys" in content
+        assert f"sys.path.insert(0, '{bundle_path}')" in content
+        assert f"os.environ.get('{condition_env}', '').lower() == 'true'" in content
+
+    def test_import_package_bundle_with_priority(self, venv):
+        """Test that priority=True uses sys.path.insert(0, ...)."""
+        bundle_path = "/usr/local/lib/python3.11/site-packages"
+
+        venv.import_package_bundle(bundle_path, priority=True)
+
+        pth_file = Path(venv.site_packages_path) / "deepnote.pth"
+        content = pth_file.read_text()
+
+        assert "import sys" in content
+        assert f"sys.path.insert(0, '{bundle_path}')" in content
+        assert "os.environ.get" not in content
+
+    def test_import_package_bundle_ordering(self, venv):
+        """Test .pth file content matches expected order from __main__.py."""
+        server_libs = "/tmp/python3.11/server-libs/lib/python3.11/site-packages"
+        system_site = "/usr/local/lib/python3.11/site-packages"
+        kernel_libs = "/tmp/python3.11/kernel-libs/lib/python3.11/site-packages"
+
+        venv.import_package_bundle(
+            server_libs, condition_env="DEEPNOTE_INCLUDE_SERVER_PACKAGES"
+        )
+        venv.import_package_bundle(system_site, priority=True)
+        venv.import_package_bundle(kernel_libs)
+
+        pth_file = Path(venv.site_packages_path) / "deepnote.pth"
+        lines = pth_file.read_text().splitlines()
+
+        assert len(lines) == 3
+        # Line 1: server-libs conditional
+        assert "DEEPNOTE_INCLUDE_SERVER_PACKAGES" in lines[0]
+        assert f"sys.path.insert(0, '{server_libs}')" in lines[0]
+        # Line 2: system with priority
+        assert f"sys.path.insert(0, '{system_site}')" in lines[1]
+        # Line 3: kernel plain path
+        assert lines[2].strip() == kernel_libs
+
+    def test_import_package_bundle_condition_env_and_priority_mutually_exclusive(
+        self, venv
+    ):
+        """Test that condition_env and priority cannot be used together."""
+        bundle_path = "/some/bundle/site-packages"
+
+        with pytest.raises(ValueError, match="mutually exclusive"):
+            venv.import_package_bundle(
+                bundle_path,
+                condition_env="SOME_ENV_VAR",
+                priority=True,
+            )