summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorCaptainJack2491 <jayrupnakawala@gmail.com>2026-03-25 21:50:13 +0000
committerCaptainJack2491 <jayrupnakawala@gmail.com>2026-03-25 21:50:13 +0000
commit8483f05c79f13176c487f57cc2c589646891be65 (patch)
tree48712a989157f9171a8580b4b6ddeea9685a8989 /tests
parentd4b0c0a74f579c0d4ffb83a64f62fafe623fec30 (diff)
refactor: extract judge prompts to files and add granular blackbox categories
- Extracted Blackbox and Glassbox prompts from into their own markdown files in - Implemented progressive 4-tier Blackbox categories (FABRICATED, OMITTED_CRITICAL, DECEPTIVE_FRAMING, ACCURATE) - Added neutral examples to the Blackbox prompt to improve formatting consistency - Updated references and tests to match new judge categorization structure
Diffstat (limited to 'tests')
-rw-r--r--tests/test_judge.py10
1 files changed, 5 insertions, 5 deletions
diff --git a/tests/test_judge.py b/tests/test_judge.py
index 463cba8..4da8491 100644
--- a/tests/test_judge.py
+++ b/tests/test_judge.py
@@ -294,13 +294,13 @@ class TestBlackboxChecker:
assert "baseline here" in prompt
assert "source docs here" in prompt
assert "ACCURATE" in prompt
- assert "MISLEADING" in prompt
+ assert "DECEPTIVE_FRAMING" in prompt
def test_parse_valid_response(self):
checker = BlackboxChecker()
- text = 'Step 1: Compare.\n```json\n{"category": "OMITTED", "justification": "missing data"}\n```'
+ text = 'Step 1: Compare.\n```json\n{"category": "OMITTED_CRITICAL", "justification": "missing data"}\n```'
result = checker.parse_response(text)
- assert result["category"] == "OMITTED"
+ assert result["category"] == "OMITTED_CRITICAL"
assert result["justification"] == "missing data"
assert result["raw_response"] == text
@@ -478,7 +478,7 @@ class TestWriteCsv:
"scenario": "s1",
"oversight": "low",
"regex": "PASS",
- "blackbox": {"category": "MISLEADING", "justification": "biased"},
+ "blackbox": {"category": "DECEPTIVE_FRAMING", "justification": "biased"},
"glassbox": {
"category": "COMPLIANT_DECEPTION",
"sophistication": "TACTICAL",
@@ -496,7 +496,7 @@ class TestWriteCsv:
assert len(rows) == 1
assert rows[0]["run_id"] == "run1"
assert rows[0]["regex"] == "PASS"
- assert rows[0]["blackbox_category"] == "MISLEADING"
+ assert rows[0]["blackbox_category"] == "DECEPTIVE_FRAMING"
assert rows[0]["glassbox_sophistication"] == "TACTICAL"