From 80f4f96758a1ba45c13361c2f87bff6fa7de8fe2 Mon Sep 17 00:00:00 2001 From: CaptainJack2491 Date: Mon, 2 Mar 2026 15:58:19 +0000 Subject: fix: global oversight_levels now filters scenario-available levels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously a scenario with its own oversight/ directory would ignore the global oversight_levels config. Now the global list acts as a filter — only levels present in BOTH the scenario dir AND the global config are run. Warns if no levels match. --- src/runner.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/runner.py b/src/runner.py index d856ed6..f636d7f 100644 --- a/src/runner.py +++ b/src/runner.py @@ -42,8 +42,15 @@ class ExperimentRunner: skipped_runs = 0 for model_config in self.config.models: for scenario_config in self.config.scenarios: - # Use per‑scenario oversight levels if defined, otherwise fall back to global list - oversight_levels = scenario_config.oversight_levels or self.config.oversight_levels + # Scenario may define available oversight levels (from oversight/ dir). + # Global oversight_levels in config acts as a FILTER on what actually runs. + available = scenario_config.oversight_levels or self.config.oversight_levels + global_filter = self.config.oversight_levels + oversight_levels = [lvl for lvl in available if lvl in global_filter] + if not oversight_levels: + logger.warning(f"No matching oversight levels for {scenario_config.path}. " + f"Available: {available}, Config filter: {global_filter}") + continue for oversight_level in oversight_levels: runs, skipped = self._run_combo(model_config, scenario_config, oversight_level) total_runs += runs -- cgit v1.2.3