Fix JSON serialization for checkpoint URIs and Inspect AI imports

Author: haasonsaas

inspect_eval.py

@@ -10,9 +10,21 @@
     from inspect_ai.model import GenerateConfig, Model
     from inspect_ai.scorer import match, includes
     from inspect_ai.solver import generate
+    
+    QA_SAMPLES = [
+        Sample(input="What is 2 + 2?", target="4"),
+        Sample(input="What is the capital of France?", target="Paris"),
+        Sample(input="What color is grass?", target="green"),
+        Sample(input="How many days in a week?", target="7"),
+        Sample(input="What is 10 x 5?", target="50"),
+    ]
+    
     INSPECT_AVAILABLE = True
 except ImportError:
     INSPECT_AVAILABLE = False
+    Task = None
+    task = None
+    QA_SAMPLES = []
 
 try:
     from tinker_cookbook.eval.inspect_utils import InspectAPIFromTinkerSampling
@@ -21,31 +33,25 @@
     TINKER_INSPECT_AVAILABLE = False
 
 
-QA_SAMPLES = [
-    Sample(input="What is 2 + 2?", target="4"),
-    Sample(input="What is the capital of France?", target="Paris"),
-    Sample(input="What color is grass?", target="green"),
-    Sample(input="How many days in a week?", target="7"),
-    Sample(input="What is 10 x 5?", target="50"),
-]
-
-
-@task
-def simple_qa_task() -> Task:
+def simple_qa_task():
     """
     Simple QA evaluation task for demo purposes.
     
     Tests basic factual knowledge with exact match scoring.
     """
-    if not INSPECT_AVAILABLE:
+    if not INSPECT_AVAILABLE or not Task:
         raise ImportError("inspect_ai required for this task")
 
-    return Task(
-        name="simple_qa",
-        dataset=MemoryDataset(name="simple_qa", samples=QA_SAMPLES),
-        solver=generate(),
-        scorer=includes(),
-    )
+    @task
+    def _simple_qa() -> Task:
+        return Task(
+            name="simple_qa",
+            dataset=MemoryDataset(name="simple_qa", samples=QA_SAMPLES),
+            solver=generate(),
+            scorer=includes(),
+        )
+    
+    return _simple_qa()
 
 
 async def run_inspect_evaluation(

logger.py

@@ -85,7 +85,7 @@ def log_checkpoint(self, round_num: int, checkpoint_uri: str):
         """Log a checkpoint save."""
         self.log_event("checkpoint", {
             "round": round_num,
-            "checkpoint_uri": checkpoint_uri,
+            "checkpoint_uri": str(checkpoint_uri),
         })
 
     def log_config(self, config: Dict[str, Any]):

runs/20251001_185228/metrics.jsonl

@@ -0,0 +1,4 @@
+{"timestamp": "2025-10-01T18:52:28.055140", "run_id": "20251001_185228", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-39/test_early_stopping_on_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 5, "lr_decay": 0.8, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:28.079142", "run_id": "20251001_185228", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-39/test_full_rounds_below_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.9, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:28.090092", "run_id": "20251001_185228", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-39/test_evalops_integration_calle0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 1, "lr_decay": 0.8, "evalops_enabled": true, "evalops_test_suite_id": "suite-123", "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:28.101579", "run_id": "20251001_185228", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-39/test_lr_decay_across_rounds0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 1.0, "eval_threshold": 0.99, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 0, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}

runs/20251001_185238/metrics.jsonl

@@ -0,0 +1,8 @@
+{"timestamp": "2025-10-01T18:52:38.285251", "run_id": "20251001_185238", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-40/test_early_stopping_on_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 5, "lr_decay": 0.8, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:38.285705", "run_id": "20251001_185238", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4462745920'>"}
+{"timestamp": "2025-10-01T18:52:38.307962", "run_id": "20251001_185238", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-40/test_full_rounds_below_thresho0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.9, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:38.308338", "run_id": "20251001_185238", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4462753312'>"}
+{"timestamp": "2025-10-01T18:52:38.321016", "run_id": "20251001_185238", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-40/test_evalops_integration_calle0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 0.0001, "eval_threshold": 0.8, "max_rounds": 1, "lr_decay": 0.8, "evalops_enabled": true, "evalops_test_suite_id": "suite-123", "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 100, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:38.321513", "run_id": "20251001_185238", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4462744912'>"}
+{"timestamp": "2025-10-01T18:52:38.332722", "run_id": "20251001_185238", "event": "config", "base_model": "test-model", "train_file": "/private/var/folders/1s/sm2bn6kd64gdnv757t9qshcm0000gn/T/pytest-of-jonathanhaas/pytest-40/test_lr_decay_across_rounds0/train.jsonl", "eval_tasks": [], "renderer_name": "default", "learning_rate": 1.0, "eval_threshold": 0.99, "max_rounds": 3, "lr_decay": 0.5, "evalops_enabled": false, "evalops_test_suite_id": null, "evalops_api_url": null, "steps_per_round": 1, "batch_size": 8, "max_seq_length": 2048, "lora_rank": 16, "warmup_steps": 0, "max_steps": 1000, "min_lr": 1e-06, "use_recommended_lr": false}
+{"timestamp": "2025-10-01T18:52:38.333083", "run_id": "20251001_185238", "event": "checkpoint", "round": 1, "checkpoint_uri": "<MagicMock name='mock.create_lora_training_client().save_weights_for_sampler().result().path' id='4468904960'>"}

runs/20251001_185238/run_state.json

@@ -0,0 +1,5 @@
+{
+  "round_idx": 1,
+  "global_step": 1,
+  "learning_rate": 1.0,
+  "checkpoint_uri":