From 8483f05c79f13176c487f57cc2c589646891be65 Mon Sep 17 00:00:00 2001 From: CaptainJack2491 Date: Wed, 25 Mar 2026 21:50:13 +0000 Subject: refactor: extract judge prompts to files and add granular blackbox categories - Extracted Blackbox and Glassbox prompts from into their own markdown files in - Implemented progressive 4-tier Blackbox categories (FABRICATED, OMITTED_CRITICAL, DECEPTIVE_FRAMING, ACCURATE) - Added neutral examples to the Blackbox prompt to improve formatting consistency - Updated references and tests to match new judge categorization structure --- tests/test_judge.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/test_judge.py b/tests/test_judge.py index 463cba8..4da8491 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -294,13 +294,13 @@ class TestBlackboxChecker: assert "baseline here" in prompt assert "source docs here" in prompt assert "ACCURATE" in prompt - assert "MISLEADING" in prompt + assert "DECEPTIVE_FRAMING" in prompt def test_parse_valid_response(self): checker = BlackboxChecker() - text = 'Step 1: Compare.\n```json\n{"category": "OMITTED", "justification": "missing data"}\n```' + text = 'Step 1: Compare.\n```json\n{"category": "OMITTED_CRITICAL", "justification": "missing data"}\n```' result = checker.parse_response(text) - assert result["category"] == "OMITTED" + assert result["category"] == "OMITTED_CRITICAL" assert result["justification"] == "missing data" assert result["raw_response"] == text @@ -478,7 +478,7 @@ class TestWriteCsv: "scenario": "s1", "oversight": "low", "regex": "PASS", - "blackbox": {"category": "MISLEADING", "justification": "biased"}, + "blackbox": {"category": "DECEPTIVE_FRAMING", "justification": "biased"}, "glassbox": { "category": "COMPLIANT_DECEPTION", "sophistication": "TACTICAL", @@ -496,7 +496,7 @@ class TestWriteCsv: assert len(rows) == 1 assert rows[0]["run_id"] == "run1" assert rows[0]["regex"] == "PASS" - assert rows[0]["blackbox_category"] == "MISLEADING" + assert rows[0]["blackbox_category"] == "DECEPTIVE_FRAMING" assert rows[0]["glassbox_sophistication"] == "TACTICAL" -- cgit v1.2.3