Sub topic (#43)

* adding refactored task generation. updated prompts to ask for json outputs, and updated corresponding output parser.

* fixed retry, json processing, and max token.

* switichin to two phase task generation.

* switichin to two phase task generation. part 2.

* updated agentic config and readme.

* simplified task generations.

* simplified task generation.

* fixed mypy errors.

* ruff fix.

* updated saved file name for solutions.

* added extra details to agent solution messages.

* fixed prompts.

* fixed output dir name to include area name.

* fixed task solver output dir name.

* upgraded json handling, and model call.

* updated readme to include latest agentic changes.

* task diversity study scripts added.

* updated prompts, and combination logic.

* added resume, and retry.

Author: kohankhaki

experimental/diverse_task_config.yaml

@@ -0,0 +1,37 @@
+# Configuration for Diverse Task Generator
+
+# Model settings
+model:
+  name: gpt-4o  # OpenAI model to use
+  temperature: 1.0  # Temperature for all steps
+  max_tokens: 8192  # Max tokens for all steps
+  max_retries: 3  # Number of retry attempts for API calls
+  retry_delay: 2.0  # Initial delay between retries in seconds (exponential backoff)
+
+# Task generation settings
+generation:
+  tasks_per_blueprint: 3  # Number of tasks to generate per blueprint
+  min_subtopics: 3  # Suggested minimum number of sub-topics
+  max_subtopics: 8  # Suggested maximum number of sub-topics
+
+# Output settings
+output:
+  base_dir: diverse_task_outputs
+  save_intermediate_steps: true  # Save each step's output
+  pretty_print_json: true  # Indent JSON files
+
+# Input settings
+input:
+  capability_json_path: capability.json  # Default capability JSON file path
+
+# Verification criteria
+verification:
+  pass_threshold: 0.8  # Minimum pass rate to consider successful
+  strict_mode: false  # If true, all alignment criteria must pass
+
+# Example capability for quick testing
+example_capability:
+  name: "compound_interest_calculations"
+  description: "The ability to calculate compound interest for various scenarios, including different compounding frequencies (annually, semi-annually, quarterly, monthly), different time periods, and understanding how changes in principal, rate, or time affect the final amount."
+  domain: "personal_finance"
+  area: "investing_and_savings"

experimental/diverse_task_constants.py

@@ -0,0 +1,34 @@
+"""Constants for diverse task generation."""
+
+BLOOMS_TAXONOMY = {
+    "Remember": {
+        "description": "Recall or recognize facts, terms, and basic concepts. Example verbs: define, list, identify."
+    },
+    "Understand": {
+        "description": "Explain ideas or concepts and interpret information in one's own words. Example verbs: summarize, describe, classify."
+    },
+    "Apply": {
+        "description": "Use knowledge or methods in new but familiar situations. Example verbs: calculate, demonstrate, use, implement."
+    },
+    "Analyze": {
+        "description": "Break information into parts and examine relationships or patterns. Example verbs: differentiate, compare, examine, infer."
+    },
+    "Evaluate": {
+        "description": "Make judgments based on criteria and standards. Example verbs: justify, critique, assess, argue."
+    },
+    "Create": {
+        "description": "Combine elements to form a new pattern, structure, or product. Example verbs: design, compose, formulate, generate."
+    },
+}
+
+DIFFICULTY_LEVELS = {
+    "easy": {
+        "description": "Involves direct recall, recognition, or simple application of knowledge and procedures."
+    },
+    "medium": {
+        "description": "Requires connecting multiple ideas, performing multi-step reasoning, or applying knowledge in new but familiar contexts."
+    },
+    "hard": {
+        "description": "Involves complex reasoning, integration of several sub-topics, or solving non-trivial problems that demand deeper conceptual understanding."
+    },
+}

experimental/diverse_task_dataclasses.py

@@ -0,0 +1,77 @@
+"""Dataclasses for the diverse task generation pipeline."""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+
+@dataclass
+class Capability:
+    """Represents a capability to be tested."""
+
+    name: str
+    description: str
+    domain: str
+    area: Optional[str] = None
+    example_tasks: List[Dict] = field(default_factory=list)
+
+
+@dataclass
+class SubTopic:
+    """Represents a sub-topic within a capability."""
+
+    name: str
+    description: Optional[str] = None
+
+
+@dataclass
+class Combination:
+    """Represents a valid (content, difficulty, reasoning) combination."""
+
+    content: str
+    difficulty: str
+    reasoning: str
+    rationale: Optional[str] = None
+
+
+@dataclass
+class Blueprint:
+    """Represents a task blueprint for a specific combination."""
+
+    combination_id: int
+    subtopic: str
+    difficulty: str
+    reasoning: str
+    blueprint: str
+    key_characteristics: List[str] = field(default_factory=list)
+    example_question_outline: Optional[str] = None
+    rationale: Optional[str] = None
+
+
+@dataclass
+class Task:
+    """Represents a generated multiple-choice task."""
+
+    task_id: str
+    blueprint_id: int
+    subtopic: str
+    difficulty: str
+    reasoning: str
+    question: str
+    choices: Dict[str, str]
+    correct_answer: str
+    explanation: Optional[str] = None
+    alignment_notes: Optional[str] = None
+
+
+@dataclass
+class VerificationResult:
+    """Represents the verification result for a task."""
+
+    task_id: str
+    subtopic_aligned: bool
+    difficulty_aligned: bool
+    reasoning_aligned: bool
+    choices_appropriate: bool
+    overall_aligned: bool
+    feedback: str
+    suggested_improvements: Optional[str] = None