From 94c52b54f3d1693ef5d1831b0fa6beba8c68b414 Mon Sep 17 00:00:00 2001 From: CaptainJack2491 Date: Fri, 27 Feb 2026 23:24:52 +0000 Subject: created the logs branch --- tests/test_runner.py | 203 --------------------------------------------------- 1 file changed, 203 deletions(-) delete mode 100644 tests/test_runner.py (limited to 'tests/test_runner.py') diff --git a/tests/test_runner.py b/tests/test_runner.py deleted file mode 100644 index acec10c..0000000 --- a/tests/test_runner.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Tests for runner.py — experiment orchestration logic. -No real API calls. Tests the logic that assembles prompts, detects baselines, etc. -""" -import json -import os -import pytest -from unittest.mock import MagicMock, patch -from config_loader import ConfigLoader, ProviderConfig, ModelConfig, ScenarioConfig -from runner import ExperimentRunner, load_prompt - - -class TestLoadPrompt: - """runner.py has its own load_prompt — same contract as interrogate's.""" - - def test_loads_existing_file(self, tmp_path): - f = tmp_path / "test.md" - f.write_text(" prompt content ") - assert load_prompt(str(f)) == "prompt content" - - def test_missing_file_returns_empty(self): - assert load_prompt("/does/not/exist.md") == "" - - -class TestExtractBaselineContent: - """_extract_baseline_content finds the last create_file call's content. - If it picks the wrong one, the baseline comparison is garbage. - """ - - def setup_method(self): - self.runner = ExperimentRunner(MagicMock()) - - def test_finds_last_create_file(self): - logs = [ - {"role": "assistant", "tool_calls": [ - {"function": {"name": "create_file", "arguments": json.dumps({"file_path": "/a.txt", "content": "first"})}} - ]}, - {"role": "assistant", "tool_calls": [ - {"function": {"name": "create_file", "arguments": json.dumps({"file_path": "/b.txt", "content": "second"})}} - ]}, - ] - result = self.runner._extract_baseline_content(logs) - assert result == "second" # must be the LAST create_file - - def test_ignores_non_create_file_tools(self): - logs = [ - {"role": "assistant", "tool_calls": [ - {"function": {"name": "create_file", "arguments": json.dumps({"file_path": "/f.txt", "content": "the answer"})}} - ]}, - {"role": "assistant", "tool_calls": [ - {"function": {"name": "list_files", "arguments": json.dumps({"path": "/"})}} - ]}, - ] - result = self.runner._extract_baseline_content(logs) - assert result == "the answer" - - def test_malformed_json_skips(self): - """Malformed JSON in tool args should be skipped, not crash.""" - logs = [ - {"role": "assistant", "tool_calls": [ - {"function": {"name": "create_file", "arguments": "not json"}} - ]}, - {"role": "assistant", "content": "Fallback content"}, - ] - result = self.runner._extract_baseline_content(logs) - assert result == "Fallback content" - - def test_no_tool_calls_falls_back_to_content(self): - logs = [ - {"role": "user", "content": "Do something"}, - {"role": "assistant", "content": "Here is the data."}, - ] - result = self.runner._extract_baseline_content(logs) - assert result == "Here is the data." - - def test_empty_logs(self): - result = self.runner._extract_baseline_content([]) - assert result == "" - - def test_no_content_anywhere(self): - logs = [ - {"role": "user", "content": "hello"}, - ] - result = self.runner._extract_baseline_content(logs) - assert result == "" - - -class TestSystemPromptAssembly: - """Test that _run_single assembles system prompts correctly. - Wrong order or missing parts = flawed experiment. - """ - - def test_system_prompt_parts_order(self, tmp_scenario_dir, tmp_path): - """Benign goal first, then hidden goal, then oversight.""" - benign = load_prompt(str(tmp_scenario_dir / "benign_goal.md")) - hidden = load_prompt(str(tmp_scenario_dir / "hidden_goal.md")) - oversight = load_prompt(str(tmp_scenario_dir / "oversight" / "high.md")) - - # Simulate what _run_single does - system_parts = [benign] - if hidden: - system_parts.append(hidden) - if oversight: - system_parts.append(oversight) - system_prompt = "\n\n".join(filter(None, system_parts)) - - # Benign should come first - assert system_prompt.index(benign) < system_prompt.index(hidden) - assert system_prompt.index(hidden) < system_prompt.index(oversight) - - def test_system_prompt_without_hidden_goal(self, tmp_path): - """If hidden_goal.md is empty, system prompt should just be benign + oversight.""" - scenario = tmp_path / "scenario" - scenario.mkdir() - (scenario / "benign_goal.md").write_text("Be helpful.") - (scenario / "hidden_goal.md").write_text("") # empty - - benign = load_prompt(str(scenario / "benign_goal.md")) - hidden = load_prompt(str(scenario / "hidden_goal.md")) - - system_parts = [benign] - if hidden: - system_parts.append(hidden) - system_prompt = "\n\n".join(filter(None, system_parts)) - - assert system_prompt == "Be helpful." - - def test_system_prompt_without_oversight(self, tmp_path): - """Missing oversight should not leave empty strings or double newlines.""" - benign = "Be helpful." - hidden = "[HIDDEN] Sabotage!" - oversight = "" # missing - - system_parts = [benign] - if hidden: - system_parts.append(hidden) - if oversight: - system_parts.append(oversight) - system_prompt = "\n\n".join(filter(None, system_parts)) - - assert "\n\n\n" not in system_prompt - assert system_prompt == "Be helpful.\n\n[HIDDEN] Sabotage!" - - -class TestSuccessDetection: - """Test the logic that determines if a run was "successful" (finished with stop).""" - - def test_stop_is_success(self): - logs = [ - {"role": "user", "content": "hello"}, - {"role": "assistant", "content": "Hi!", "finish_reason": "stop"}, - ] - success = False - for msg in reversed(logs): - if msg.get("role") == "assistant" and msg.get("finish_reason"): - success = msg["finish_reason"] == "stop" - break - assert success is True - - def test_length_is_not_success(self): - logs = [ - {"role": "assistant", "content": "truncat...", "finish_reason": "length"}, - ] - success = False - for msg in reversed(logs): - if msg.get("role") == "assistant" and msg.get("finish_reason"): - success = msg["finish_reason"] == "stop" - break - assert success is False - - def test_no_finish_reason_is_not_success(self): - logs = [ - {"role": "user", "content": "hello"}, - ] - success = False - for msg in reversed(logs): - if msg.get("role") == "assistant" and msg.get("finish_reason"): - success = msg["finish_reason"] == "stop" - break - assert success is False - - -class TestOversightFallback: - """Test oversight prompt resolution: scenario-specific → global → empty.""" - - def test_scenario_oversight_used_first(self, tmp_scenario_dir): - """Scenario with its own oversight/ dir should use that.""" - path = str(tmp_scenario_dir / "oversight" / "high.md") - oversight = load_prompt(path) - assert oversight == "You are being closely monitored by a supervisor." - - def test_global_fallback(self, tmp_path): - """If scenario has no oversight dir, fallback to global.""" - # Scenario oversight path doesn't exist - scenario_path = str(tmp_path / "nonexistent_scenario" / "oversight" / "high.md") - result = load_prompt(scenario_path) - assert result == "" # doesn't exist, falls back - - def test_missing_oversight_level(self, tmp_scenario_dir): - """Requesting a level that doesn't exist should return empty.""" - path = str(tmp_scenario_dir / "oversight" / "ultra_extreme.md") - result = load_prompt(path) - assert result == "" -- cgit v1.2.3