summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorCaptainJack2491 <jayrupnakawala@gmail.com>2026-02-27 13:43:52 +0000
committerCaptainJack2491 <jayrupnakawala@gmail.com>2026-02-27 13:45:41 +0000
commitcfc25ef6112d08c6d921965d685d2831fb705604 (patch)
tree0725a4eca607feddb8ca418331bb390f52f9ba6b /src
parentdac18577b9a48884da61882e059b12d147fb0228 (diff)
fix: separate skipped runs from incomplete in summary output
Diffstat (limited to 'src')
-rw-r--r--src/runner.py14
1 files changed, 9 insertions, 5 deletions
diff --git a/src/runner.py b/src/runner.py
index 3a03c8c..22b16aa 100644
--- a/src/runner.py
+++ b/src/runner.py
@@ -35,20 +35,24 @@ class ExperimentRunner:
print(f"{'='*60}\n")
total_runs = 0
+ skipped_runs = 0
for model_config in self.config.models:
for scenario_config in self.config.scenarios:
# Use per‑scenario oversight levels if defined, otherwise fall back to global list
oversight_levels = scenario_config.oversight_levels or self.config.oversight_levels
for oversight_level in oversight_levels:
- runs = self._run_combo(model_config, scenario_config, oversight_level)
+ runs, skipped = self._run_combo(model_config, scenario_config, oversight_level)
total_runs += runs
+ skipped_runs += skipped
# Summary
successful = sum(1 for r in self.results if r.get("success", False))
- incomplete = total_runs - successful
+ new_runs = total_runs - skipped_runs
+ incomplete = new_runs - successful
print(f"\n{'='*60}")
- print(f"Experiment Complete: {total_runs} runs")
+ print(f"Experiment Complete: {total_runs} total runs")
+ print(f" SKIPPED: {skipped_runs}")
print(f" SUCCESS: {successful}")
print(f" INCOMPLETE: {incomplete}")
print(f"{'='*60}\n")
@@ -86,7 +90,7 @@ class ExperimentRunner:
if existing_runs >= scenario_config.runs:
print(f" SKIP: {existing_runs}/{scenario_config.runs} runs already exist")
- return existing_runs
+ return existing_runs, existing_runs
elif existing_runs > 0:
print(f" RESUME: {existing_runs}/{scenario_config.runs} runs already exist, continuing from run {existing_runs + 1}")
@@ -107,7 +111,7 @@ class ExperimentRunner:
traceback.print_exc()
print(f"--- Completed: {runs_completed}/{scenario_config.runs} runs ---")
- return runs_completed
+ return runs_completed, existing_runs
def _extract_baseline_content(self, logs: List[Dict]) -> str:
"""Extract baseline content from the last create_file tool call."""