1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
"""
Judge checker classes for the judge system.
"""
import os
import re
from typing import Any, Dict, List
import yaml
from .prompts import (
BLACKBOX_CATEGORIES,
BLACKBOX_PROMPT,
GLASSBOX_CATEGORIES,
GLASSBOX_PROMPT,
SOPHISTICATION_TIERS,
)
from .helpers import parse_json_verdict
class RegexChecker:
"""Deterministic string/pattern matching on model output."""
def __init__(self, rules_path: str):
self.rules = self._load_rules(rules_path)
@staticmethod
def _load_rules(rules_path: str) -> List[Dict]:
"""Load regex rules from a YAML file.
Supports two formats:
- A dict with a 'rules' key: {rules: [...]}
- A flat list of rules: [...]
"""
if not os.path.exists(rules_path):
return []
with open(rules_path, "r") as f:
data = yaml.safe_load(f)
if not data:
return []
if isinstance(data, list):
return data
return data.get("rules", []) if isinstance(data, dict) else []
def check(self, model_output: str) -> Dict[str, Any]:
"""Run all regex rules against the model output.
Returns:
{"verdict": "6/7", "matched": 6, "total": 7, "details": [...]}
"""
if not self.rules:
return {
"verdict": "PASS",
"matched": 0,
"total": 0,
"details": [{"note": "no regex rules defined"}],
"fraction": "0/0",
}
details = []
required_total = 0
required_matched = 0
for rule in self.rules:
pattern = rule.get("pattern", "")
name = rule.get("name", "unnamed")
required = rule.get("required", False)
description = rule.get("description", "")
matched = bool(re.search(pattern, model_output))
detail = {
"name": name,
"matched": matched,
"required": required,
"description": description,
}
details.append(detail)
if required:
required_total += 1
if matched:
required_matched += 1
if required_total == 0:
verdict = "PASS"
else:
verdict = "PASS" if required_matched == required_total else "FAIL"
return {
"verdict": verdict,
"matched": required_matched,
"total": required_total,
"details": details,
"fraction": f"{required_matched}/{required_total}",
}
class BlackboxChecker:
"""LLM judge comparing model output against baseline (no reasoning trace)."""
def __init__(self, model: str = "claude-sonnet-4-20250514", temperature: float = 0):
self.model = model
self.temperature = temperature
def build_prompt(
self,
model_output: str,
baseline: str,
source_documents: str,
task_description: str = "",
) -> str:
"""Build the blackbox judge prompt."""
return BLACKBOX_PROMPT.format(
task_description=task_description or "(No task description available)",
source_documents=source_documents,
baseline=baseline,
model_output=model_output,
)
def parse_response(self, response_text: str) -> Dict[str, Any]:
"""Parse the judge's response into a structured verdict."""
verdict = parse_json_verdict(response_text)
category = verdict.get("category", "UNKNOWN")
if category not in BLACKBOX_CATEGORIES:
category = "UNKNOWN"
return {
"category": category,
"justification": verdict.get("justification", ""),
"raw_response": response_text,
}
class GlassboxChecker:
"""LLM judge evaluating reasoning trace + output + system prompt for intent."""
def __init__(self, model: str = "claude-sonnet-4-20250514", temperature: float = 0):
self.model = model
self.temperature = temperature
def build_prompt(
self,
system_prompt: str,
reasoning_trace: str,
final_output: str,
) -> str:
"""Build the glassbox judge prompt."""
return GLASSBOX_PROMPT.format(
system_prompt=system_prompt,
reasoning_trace=reasoning_trace,
final_output=final_output,
)
def parse_response(self, response_text: str) -> Dict[str, Any]:
"""Parse the judge's response into a structured verdict."""
verdict = parse_json_verdict(response_text)
category = verdict.get("category", "UNKNOWN")
sophistication = verdict.get("sophistication", "UNKNOWN")
if category not in GLASSBOX_CATEGORIES:
category = "UNKNOWN"
if sophistication not in SOPHISTICATION_TIERS:
sophistication = "UNKNOWN"
return {
"category": category,
"sophistication": sophistication,
"justification": verdict.get("justification", ""),
"raw_response": response_text,
}
|
