From 239ec5c0660d2b618cefa35b0a6a1599279d533c Mon Sep 17 00:00:00 2001 From: CaptainJack2491 Date: Mon, 2 Mar 2026 15:23:50 +0000 Subject: feat: add generate_baseline toggle to config - New 'generate_baseline' option in defaults (default: true) - When false, skips baseline generation entirely - Warns if no baseline exists when generation is disabled - Updated config.yaml.example and README.md with docs --- src/config_loader.py | 5 +++++ src/runner.py | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/config_loader.py b/src/config_loader.py index 8db50cb..2badfbb 100644 --- a/src/config_loader.py +++ b/src/config_loader.py @@ -146,6 +146,11 @@ class ConfigLoader: """Get default configuration.""" return self._config.get('defaults', {}) + @property + def generate_baseline(self) -> bool: + """Whether to generate baselines before experiment runs.""" + return self._config.get('defaults', {}).get('generate_baseline', True) + @property def output_dir(self) -> str: """Get output directory.""" diff --git a/src/runner.py b/src/runner.py index 8556ed6..5cdb807 100644 --- a/src/runner.py +++ b/src/runner.py @@ -77,7 +77,14 @@ class ExperimentRunner: # Ensure baseline exists before running hidden-goal experiments output_dir = self.config.output_dir baseline_path = os.path.join(output_dir, model_name_safe, scenario_name, "baseline.md") - if not os.path.exists(baseline_path): + if not self.config.generate_baseline: + if not os.path.exists(baseline_path): + logger.warning(f"Baseline generation is DISABLED (generate_baseline: false). " + f"No baseline exists for {model_name} | {scenario_name}. " + f"Black-box judging will not be possible for these runs.") + else: + logger.info(f"\n--- Baseline exists (generation disabled): {model_name} | {scenario_name} ---") + elif not os.path.exists(baseline_path): logger.info(f"\n--- Generating baseline: {model_name} | {scenario_name} ---") self._run_baseline(model_config, provider_config, scenario_config) logger.info(f" Baseline saved to {baseline_path}") -- cgit v1.2.3