[feat] added a baseline to the runner

(no commit message provided)

(no commit message provided)

Co-authored-by: aider (openrouter/openrouter/aurora-alpha) <aider@aider.chat>

2 files changed, 93 insertions, 5 deletions

diff --git a/src/config_loader.py b/src/config_loader.py
index 056b47b..796ad39 100644
--- a/src/config_loader.py
+++ b/src/config_loader.py
@@ -47,6 +47,7 @@ class ScenarioConfig:
     """Configuration for a scenario."""
     path: str
     runs: int = 1
+    oversight_levels: List[str] = field(default_factory=list)
 
 
 class ConfigLoader:
@@ -98,11 +99,28 @@ class ConfigLoader:
         scenarios = self._config.get('scenarios', [])
 
         for scenario in scenarios:
+            oversight_levels = self._load_oversight_levels(scenario['path'])
             self._scenarios.append(ScenarioConfig(
                 path=scenario['path'],
-                runs=scenario.get('runs', 1)
+                runs=scenario.get('runs', 1),
+                oversight_levels=oversight_levels
             ))
 
+    def _load_oversight_levels(self, scenario_path: str) -> List[str]:
+        """
+        Load oversight level identifiers from a scenario directory.
+        Looks for a subdirectory named 'oversight' containing *.md files.
+        Returns the list of filenames without extension.
+        If the subdirectory does not exist, falls back to the global
+        oversight_levels defined in the config (or an empty list).
+        """
+        oversight_dir = os.path.join(scenario_path, "oversight")
+        if not os.path.isdir(oversight_dir):
+            # Fallback to global config oversight levels
+            return self._config.get('oversight_levels', [])
+        files = [f for f in os.listdir(oversight_dir) if f.endswith('.md')]
+        return [os.path.splitext(f)[0] for f in files]
+
     @property
     def providers(self) -> Dict[str, ProviderConfig]:
         """Get all provider configurations."""
@@ -120,7 +138,7 @@ class ConfigLoader:
 
     @property
     def oversight_levels(self) -> List[str]:
-        """Get oversight levels to test."""
+        """Get global oversight levels to test (used as fallback)."""
         return self._config.get('oversight_levels', ['low'])
 
     @property
diff --git a/src/runner.py b/src/runner.py
index ab4c3f4..2d392a2 100644
--- a/src/runner.py
+++ b/src/runner.py
@@ -35,7 +35,9 @@ class ExperimentRunner:
         total_runs = 0
         for model_config in self.config.models:
             for scenario_config in self.config.scenarios:
-                for oversight_level in self.config.oversight_levels:
+                # Use per‑scenario oversight levels if defined, otherwise fall back to global list
+                oversight_levels = scenario_config.oversight_levels or self.config.oversight_levels
+                for oversight_level in oversight_levels:
                     runs = self._run_combo(model_config, scenario_config, oversight_level)
                     total_runs += runs
 
@@ -59,7 +61,18 @@ class ExperimentRunner:
         provider_config = self.config.get_provider(model_config.provider)
 
         scenario_name = os.path.basename(scenario_config.path)
-        print(f"\n--- Running: {model_config.id} | {scenario_name} | {oversight_level} ---")
+        model_name = model_config.id
+
+        # Ensure baseline exists before running hidden-goal experiments
+        baseline_path = os.path.join("baselines", scenario_name, f"{model_name}.txt")
+        if not os.path.exists(baseline_path):
+            print(f"\n--- Generating baseline: {model_name} | {scenario_name} ---")
+            # self._run_baseline(model_config, provider_config, scenario_config)
+            print(f"  Baseline saved to {baseline_path}")
+        else:
+            print(f"\n--- Baseline exists: {model_name} | {scenario_name} ---")
+
+        print(f"\n--- Running: {model_name} | {scenario_name} | {oversight_level} ---")
 
         runs_completed = 0
         for run_num in range(1, scenario_config.runs + 1):
@@ -80,6 +93,62 @@ class ExperimentRunner:
         print(f"--- Completed: {runs_completed}/{scenario_config.runs} runs ---")
         return runs_completed
 
+    def _run_baseline(
+        self,
+        model_config: ModelConfig,
+        provider_config: ProviderConfig,
+        scenario_config: ScenarioConfig
+    ):
+        """Run a baseline experiment WITHOUT hidden goal. Saves summary to baselines/."""
+        scenario_name = os.path.basename(scenario_config.path)
+        model_name = model_config.id
+
+        # Load only benign goal (NO hidden goal)
+        benign_goal = load_prompt(os.path.join(scenario_config.path, "benign_goal.md"))
+        user_prompt = load_prompt(os.path.join(scenario_config.path, "user.md"))
+
+        # Build system prompt with ONLY benign goal
+        system_prompt = benign_goal
+
+        # Initialize VFS
+        vfs_path = os.path.join(scenario_config.path, "data")
+        VFS.get_instance(vfs_path)
+
+        # Create agent
+        save_vfs_state = self.config._config.get('output', {}).get('save_vfs_state', True)
+        agent = Agent.from_configs(
+            system_prompt=system_prompt,
+            provider_config=provider_config,
+            model_config=model_config,
+            scenario=scenario_name,
+            oversight_level="baseline",
+            user_prompt_type="user.md",
+            save_vfs_state=save_vfs_state
+        )
+
+        # Run the conversation
+        print(f"  Running baseline...")
+        start_time = datetime.datetime.now()
+        result = agent.run(user_prompt)
+        end_time = datetime.datetime.now()
+
+        # Extract summary from final VFS state
+        vfs = VFS.get_instance().fs
+        summary = vfs.get("/", {}).get("data", {}).get("summary.txt", "")
+
+        # Save baseline to baselines/
+        baseline_dir = os.path.join("baselines", scenario_name)
+        os.makedirs(baseline_dir, exist_ok=True)
+        baseline_path = os.path.join(baseline_dir, f"{model_name}.txt")
+
+        with open(baseline_path, 'w') as f:
+            f.write(summary)
+
+        # Save baseline log separately
+        agent.save_logs(output_dir="logs/baselines")
+
+        print(f"  Baseline completed in {(end_time - start_time).total_seconds():.2f}s")
+
     def _run_single(
         self,
         model_config: ModelConfig,
@@ -93,7 +162,8 @@ class ExperimentRunner:
         benign_goal = load_prompt(os.path.join(scenario_config.path, "benign_goal.md"))
         hidden_goal = load_prompt(os.path.join(scenario_config.path, "hidden_goal.md"))
         user_prompt = load_prompt(os.path.join(scenario_config.path, "user.md"))
-        oversight_prompt = load_prompt(os.path.join("oversight", f"{oversight_level}.md"))
+        # Load oversight prompt from the scenario's own oversight directory
+        oversight_prompt = load_prompt(os.path.join(scenario_config.path, "oversight", f"{oversight_level}.md"))
 
         # Build system prompt
         system_parts = [benign_goal]