feat(judge): add selective judging and log filtering

- Implement enabled_judges parameter in Judge class to selectively run regex, blackbox, or glassbox checks.
- Add --judges CLI argument to judge_runner.py for granular control over judging pipeline.
- Add --model and --scenario filters to judge_runner.py for targeted evaluation of experiment logs.
- Optimize synchronous judging to skip LLM calls and API key checks when only local (regex) judges are enabled.
- Update Anthropic Batch API integration to support dynamic number of requests per log based on enabled judges.
- Improve console output and logging to reflect selected filters and active judge components.

2 files changed, 201 insertions, 126 deletions

diff --git a/src/judge.py b/src/judge.py
index 77fe506..a3080ef 100644
--- a/src/judge.py
+++ b/src/judge.py
@@ -451,6 +451,7 @@ class Judge:
         log_path: str,
         logs_dir: str = "logs",
         scenarios_dir: str = "scenarios",
+        enabled_judges: set = None,
     ) -> Dict[str, Any]:
         """Judge a single experiment log synchronously (non-batch).
 
@@ -458,11 +459,16 @@ class Judge:
             log_path: Path to the experiment log JSON.
             logs_dir: Root logs directory (for finding baselines).
             scenarios_dir: Root scenarios directory (for regex rules).
+            enabled_judges: Set of judges to run ('regex', 'blackbox', 'glassbox').
 
         Returns:
             Combined verdict dict.
         """
-        if not self.client:
+        if enabled_judges is None:
+            enabled_judges = {"regex", "blackbox", "glassbox"}
+
+        run_llm = bool({"blackbox", "glassbox"} & enabled_judges)
+        if run_llm and not self.client:
             raise RuntimeError("ANTHROPIC_API_KEY not set — cannot run LLM judge checks")
 
         log_data = self.load_log(log_path)
@@ -472,56 +478,63 @@ class Judge:
 
         # Extract data
         model_output = extract_model_output(conversation)
-        system_prompt = extract_system_prompt(conversation)
-        reasoning_trace = extract_reasoning_trace(conversation)
-        baseline_output, baseline_log_data = self.load_baseline(log_data, logs_dir)
 
-        # Load source docs from baseline if available, otherwise from experiment log itself
-        if baseline_log_data:
-            source_docs = self.load_source_documents(baseline_log_data)
-        else:
-            import sys
-            print(f"  [WARN] No baseline found for {scenario} — using experiment log for source docs", file=sys.stderr)
-            source_docs = self.load_source_documents(log_data)
+        # 1. Regex check
+        regex_result = {"verdict": "", "details": []}
+        if "regex" in enabled_judges:
+            scenario_path = os.path.join(scenarios_dir, scenario)
+            regex_checker = self._get_regex_checker(scenario_path)
+            regex_result = regex_checker.check(model_output)
 
-        if not baseline_output:
-            import sys
-            print(f"  [WARN] No baseline output for {scenario} — blackbox judge will compare against source docs only", file=sys.stderr)
-            baseline_output = "(No baseline available)"
+        # 2. Blackbox check
+        bb_result = {"category": "", "justification": "", "raw_response": ""}
+        if "blackbox" in enabled_judges:
+            system_prompt = extract_system_prompt(conversation)
+            baseline_output, baseline_log_data = self.load_baseline(log_data, logs_dir)
 
-        # 1. Regex check
-        scenario_path = os.path.join(scenarios_dir, scenario)
-        regex_checker = self._get_regex_checker(scenario_path)
-        regex_result = regex_checker.check(model_output)
+            if baseline_log_data:
+                source_docs = self.load_source_documents(baseline_log_data)
+            else:
+                import sys
+                print(f"  [WARN] No baseline found for {scenario} — using experiment log for source docs", file=sys.stderr)
+                source_docs = self.load_source_documents(log_data)
 
-        # Load benign goal (task description) for blackbox context
-        benign_goal_path = os.path.join(scenario_path, "benign_goal.md")
-        task_description = ""
-        if os.path.exists(benign_goal_path):
-            with open(benign_goal_path, "r", encoding="utf-8") as f:
-                task_description = f.read().strip()
+            if not baseline_output:
+                import sys
+                print(f"  [WARN] No baseline output for {scenario} — blackbox judge will compare against source docs only", file=sys.stderr)
+                baseline_output = "(No baseline available)"
 
-        # 2. Blackbox check
-        bb_prompt = self.blackbox.build_prompt(model_output, baseline_output, source_docs, task_description)
-        bb_response = self.client.messages.create(
-            model=self.model,
-            max_tokens=4096,
-            temperature=self.temperature,
-            messages=[{"role": "user", "content": bb_prompt}],
-        )
-        bb_text = bb_response.content[0].text
-        bb_result = self.blackbox.parse_response(bb_text)
+            scenario_path = os.path.join(scenarios_dir, scenario)
+            benign_goal_path = os.path.join(scenario_path, "benign_goal.md")
+            task_description = ""
+            if os.path.exists(benign_goal_path):
+                with open(benign_goal_path, "r", encoding="utf-8") as f:
+                    task_description = f.read().strip()
+
+            bb_prompt = self.blackbox.build_prompt(model_output, baseline_output, source_docs, task_description)
+            bb_response = self.client.messages.create(
+                model=self.model,
+                max_tokens=4096,
+                temperature=self.temperature,
+                messages=[{"role": "user", "content": bb_prompt}],
+            )
+            bb_text = bb_response.content[0].text
+            bb_result = self.blackbox.parse_response(bb_text)
 
         # 3. Glassbox check
-        gb_prompt = self.glassbox.build_prompt(system_prompt, reasoning_trace, model_output)
-        gb_response = self.client.messages.create(
-            model=self.model,
-            max_tokens=4096,
-            temperature=self.temperature,
-            messages=[{"role": "user", "content": gb_prompt}],
-        )
-        gb_text = gb_response.content[0].text
-        gb_result = self.glassbox.parse_response(gb_text)
+        gb_result = {"category": "", "sophistication": "", "justification": "", "raw_response": ""}
+        if "glassbox" in enabled_judges:
+            system_prompt = extract_system_prompt(conversation)
+            reasoning_trace = extract_reasoning_trace(conversation)
+            gb_prompt = self.glassbox.build_prompt(system_prompt, reasoning_trace, model_output)
+            gb_response = self.client.messages.create(
+                model=self.model,
+                max_tokens=4096,
+                temperature=self.temperature,
+                messages=[{"role": "user", "content": gb_prompt}],
+            )
+            gb_text = gb_response.content[0].text
+            gb_result = self.glassbox.parse_response(gb_text)
 
         return {
             "run_id": run_id,
@@ -543,6 +556,7 @@ class Judge:
         log_paths: List[str],
         logs_dir: str = "logs",
         scenarios_dir: str = "scenarios",
+        enabled_judges: set = None,
     ) -> tuple:
         """Prepare batch requests for multiple log files.
 
@@ -551,6 +565,9 @@ class Judge:
             - batch_requests: list of dicts for Anthropic batch API
             - metadata_map: dict mapping custom_id → metadata needed to reassemble results
         """
+        if enabled_judges is None:
+            enabled_judges = {"regex", "blackbox", "glassbox"}
+
         batch_requests = []
         metadata_map = {}
 
@@ -561,77 +578,89 @@ class Judge:
             run_id = log_data.get("run_id", os.path.basename(log_path))
 
             model_output = extract_model_output(conversation)
-            system_prompt = extract_system_prompt(conversation)
-            reasoning_trace = extract_reasoning_trace(conversation)
-            baseline_output, baseline_log_data = self.load_baseline(log_data, logs_dir)
 
-            # Load source docs from baseline if available, otherwise from experiment log
-            if baseline_log_data:
-                source_docs = self.load_source_documents(baseline_log_data)
-            else:
-                import sys
-                print(f"  [WARN] No baseline found for {scenario} — using experiment log for source docs", file=sys.stderr)
-                source_docs = self.load_source_documents(log_data)
+            # Regex check (local, no API) — always run if enabled
+            regex_result = {"verdict": "", "details": []}
+            if "regex" in enabled_judges:
+                scenario_path = os.path.join(scenarios_dir, scenario)
+                regex_checker = self._get_regex_checker(scenario_path)
+                regex_result = regex_checker.check(model_output)
 
-            if not baseline_output:
-                baseline_output = "(No baseline available)"
-
-            # Regex check (local, no API)
-            scenario_path = os.path.join(scenarios_dir, scenario)
-            regex_checker = self._get_regex_checker(scenario_path)
-            regex_result = regex_checker.check(model_output)
-
-            # Load benign goal (task description) for blackbox context
-            benign_goal_path = os.path.join(scenario_path, "benign_goal.md")
-            task_description = ""
-            if os.path.exists(benign_goal_path):
-                with open(benign_goal_path, "r", encoding="utf-8") as f:
-                    task_description = f.read().strip()
-
-            # Store metadata — custom_id must be ≤64 chars for Anthropic Batch API
+            # Store metadata
             id_hash = hashlib.sha256(run_id.encode()).hexdigest()[:8]
-            idx = len(batch_requests) // 2
-            bb_id = f"bb_{idx:03d}_{id_hash}"
-            gb_id = f"gb_{idx:03d}_{id_hash}"
-
-            metadata_map[bb_id] = {
-                "type": "blackbox",
-                "log_path": log_path,
-                "run_id": run_id,
-                "model": log_data.get("model", ""),
-                "scenario": scenario,
-                "oversight": log_data.get("oversight_level", ""),
-                "regex_result": regex_result,
-            }
-            metadata_map[gb_id] = {
-                "type": "glassbox",
-                "log_path": log_path,
-                "run_id": run_id,
-            }
+            idx = len(batch_requests) // 2 if len(enabled_judges & {"blackbox", "glassbox"}) == 2 else len(batch_requests)
 
             # Blackbox request
-            bb_prompt = self.blackbox.build_prompt(model_output, baseline_output, source_docs, task_description)
-            batch_requests.append({
-                "custom_id": bb_id,
-                "params": {
-                    "model": self.model,
-                    "max_tokens": 4096,
-                    "temperature": self.temperature,
-                    "messages": [{"role": "user", "content": bb_prompt}],
-                },
-            })
+            if "blackbox" in enabled_judges:
+                baseline_output, baseline_log_data = self.load_baseline(log_data, logs_dir)
+
+                if baseline_log_data:
+                    source_docs = self.load_source_documents(baseline_log_data)
+                else:
+                    import sys
+                    print(f"  [WARN] No baseline found for {scenario} — using experiment log for source docs", file=sys.stderr)
+                    source_docs = self.load_source_documents(log_data)
+
+                if not baseline_output:
+                    baseline_output = "(No baseline available)"
+
+                scenario_path = os.path.join(scenarios_dir, scenario)
+                benign_goal_path = os.path.join(scenario_path, "benign_goal.md")
+                task_description = ""
+                if os.path.exists(benign_goal_path):
+                    with open(benign_goal_path, "r", encoding="utf-8") as f:
+                        task_description = f.read().strip()
+
+                bb_id = f"bb_{idx:03d}_{id_hash}"
+                metadata_map[bb_id] = {
+                    "type": "blackbox",
+                    "log_path": log_path,
+                    "run_id": run_id,
+                    "model": log_data.get("model", ""),
+                    "scenario": scenario,
+                    "oversight": log_data.get("oversight_level", ""),
+                    "regex_result": regex_result,
+                }
+
+                bb_prompt = self.blackbox.build_prompt(model_output, baseline_output, source_docs, task_description)
+                batch_requests.append({
+                    "custom_id": bb_id,
+                    "params": {
+                        "model": self.model,
+                        "max_tokens": 4096,
+                        "temperature": self.temperature,
+                        "messages": [{"role": "user", "content": bb_prompt}],
+                    },
+                })
 
             # Glassbox request
-            gb_prompt = self.glassbox.build_prompt(system_prompt, reasoning_trace, model_output)
-            batch_requests.append({
-                "custom_id": gb_id,
-                "params": {
-                    "model": self.model,
-                    "max_tokens": 4096,
-                    "temperature": self.temperature,
-                    "messages": [{"role": "user", "content": gb_prompt}],
-                },
-            })
+            if "glassbox" in enabled_judges:
+                system_prompt = extract_system_prompt(conversation)
+                reasoning_trace = extract_reasoning_trace(conversation)
+
+                gb_id = f"gb_{idx:03d}_{id_hash}"
+                metadata_map[gb_id] = {
+                    "type": "glassbox",
+                    "log_path": log_path,
+                    "run_id": run_id,
+                }
+                # Also store regex result in glassbox metadata if blackbox is disabled
+                if "blackbox" not in enabled_judges:
+                    metadata_map[gb_id]["model"] = log_data.get("model", "")
+                    metadata_map[gb_id]["scenario"] = scenario
+                    metadata_map[gb_id]["oversight"] = log_data.get("oversight_level", "")
+                    metadata_map[gb_id]["regex_result"] = regex_result
+
+                gb_prompt = self.glassbox.build_prompt(system_prompt, reasoning_trace, model_output)
+                batch_requests.append({
+                    "custom_id": gb_id,
+                    "params": {
+                        "model": self.model,
+                        "max_tokens": 4096,
+                        "temperature": self.temperature,
+                        "messages": [{"role": "user", "content": gb_prompt}],
+                    },
+                })
 
         return batch_requests, metadata_map
 
diff --git a/src/judge_runner.py b/src/judge_runner.py
index cbc5e51..9f81870 100644
--- a/src/judge_runner.py
+++ b/src/judge_runner.py
@@ -16,8 +16,14 @@ sys.path.insert(0, os.path.dirname(__file__))
 from judge import Judge, extract_model_output, extract_reasoning_trace, extract_system_prompt
 
 
-def discover_log_files(logs_dir: str) -> List[str]:
-    """Find all experiment log JSON files, skipping baselines."""
+def discover_log_files(logs_dir: str, model_filter: str = None, scenario_filter: str = None) -> List[str]:
+    """Find all experiment log JSON files, skipping baselines.
+    
+    Args:
+        logs_dir: Root directory containing experiment logs.
+        model_filter: If set, only include logs from this model (e.g. 'moonshotai/kimi-k2.5').
+        scenario_filter: If set, only include logs from this scenario.
+    """
     log_files = []
     for root, dirs, files in os.walk(logs_dir):
         # Skip baseline directories
@@ -25,7 +31,17 @@ def discover_log_files(logs_dir: str) -> List[str]:
             continue
         for f in files:
             if f.endswith(".json"):
-                log_files.append(os.path.join(root, f))
+                full_path = os.path.join(root, f)
+                # Apply model filter
+                if model_filter:
+                    model_safe = model_filter.replace("/", "_")
+                    if model_safe not in full_path:
+                        continue
+                # Apply scenario filter
+                if scenario_filter:
+                    if scenario_filter not in full_path:
+                        continue
+                log_files.append(full_path)
     return sorted(log_files)
 
 
@@ -180,6 +196,21 @@ def main():
         default=30,
         help="Seconds between batch status polls (default: 30)",
     )
+    parser.add_argument(
+        "--model",
+        help="Filter logs to a specific model (e.g. 'moonshotai/kimi-k2.5')",
+    )
+    parser.add_argument(
+        "--scenario",
+        help="Filter logs to a specific scenario (e.g. 'child_protection')",
+    )
+    parser.add_argument(
+        "--judges",
+        nargs="+",
+        choices=["regex", "blackbox", "glassbox"],
+        default=["regex", "blackbox", "glassbox"],
+        help="Which judges to run (default: all three). E.g. --judges regex blackbox",
+    )
     args = parser.parse_args()
 
     # Load judge config
@@ -194,22 +225,27 @@ def main():
     if args.log_file:
         log_files = [args.log_file]
     else:
-        log_files = discover_log_files(args.logs_dir)
+        log_files = discover_log_files(args.logs_dir, model_filter=args.model, scenario_filter=args.scenario)
 
     if not log_files:
         print("No log files found to judge.")
         return
 
+    enabled_judges = set(args.judges)
+    run_llm = bool({"blackbox", "glassbox"} & enabled_judges)
+
     print(f"\n{'='*60}")
     print(f"Judging {len(log_files)} experiment log(s)")
-    print(f"  Model: {model}")
-    print(f"  Mode: {args.mode}")
+    print(f"  Judges: {', '.join(sorted(enabled_judges))}")
+    if run_llm:
+        print(f"  Judge model: {model}")
+        print(f"  Mode: {args.mode}")
     print(f"  Output: {args.output}")
     print(f"  Judge logs: {judge_log_dir}")
     print(f"{'='*60}\n")
 
-    if args.mode == "single":
-        # Synchronous mode — judge one at a time
+    if not run_llm or args.mode == "single":
+        # Regex-only mode or synchronous mode — judge one at a time
         verdicts = []
         for i, log_path in enumerate(log_files, 1):
             print(f"[{i}/{len(log_files)}] Judging: {log_path}")
@@ -218,16 +254,20 @@ def main():
                     log_path=log_path,
                     logs_dir=args.logs_dir,
                     scenarios_dir=args.scenarios_dir,
+                    enabled_judges=enabled_judges,
                 )
                 verdicts.append(verdict)
 
                 # Save judge log
                 jlog = save_judge_log(verdict, judge_log_dir, judge_model=model)
-                print(f"  → regex={verdict['regex']} "
-                      f"blackbox={verdict['blackbox']['category']} "
-                      f"glassbox={verdict['glassbox']['category']}/"
-                      f"{verdict['glassbox']['sophistication']}")
-                print(f"  → Judge log: {jlog}")
+                parts = []
+                if "regex" in enabled_judges:
+                    parts.append(f"regex={verdict['regex']}")
+                if "blackbox" in enabled_judges:
+                    parts.append(f"blackbox={verdict['blackbox']['category']}")
+                if "glassbox" in enabled_judges:
+                    parts.append(f"glassbox={verdict['glassbox']['category']}/{verdict['glassbox']['sophistication']}")
+                print(f"  → {' '.join(parts)}")
             except Exception as e:
                 print(f"  ERROR: {e}")
                 import traceback
@@ -239,14 +279,16 @@ def main():
 
     else:
         # Batch mode — use Anthropic Batch API
+        checks_per_log = len({"blackbox", "glassbox"} & enabled_judges)
         print("Preparing batch requests...")
         batch_requests, metadata_map = judge.prepare_batch_requests(
             log_paths=log_files,
             logs_dir=args.logs_dir,
             scenarios_dir=args.scenarios_dir,
+            enabled_judges=enabled_judges,
         )
 
-        print(f"  {len(batch_requests)} API requests ({len(log_files)} logs × 2 checks)")
+        print(f"  {len(batch_requests)} API requests ({len(log_files)} logs × {checks_per_log} checks)")
 
         print("Submitting batch...")
         batch_id = judge.submit_batch(batch_requests)
@@ -263,10 +305,14 @@ def main():
             jlog = save_judge_log(verdict, judge_log_dir, judge_model=model)
             bb = verdict.get("blackbox", {})
             gb = verdict.get("glassbox", {})
-            print(f"  {verdict['run_id']}: "
-                  f"regex={verdict.get('regex', '?')} "
-                  f"blackbox={bb.get('category', '?')} "
-                  f"glassbox={gb.get('category', '?')}/{gb.get('sophistication', '?')}")
+            parts = [f"{verdict['run_id']}:"]
+            if "regex" in enabled_judges:
+                parts.append(f"regex={verdict.get('regex', '?')}")
+            if "blackbox" in enabled_judges:
+                parts.append(f"blackbox={bb.get('category', '?')}")
+            if "glassbox" in enabled_judges:
+                parts.append(f"glassbox={gb.get('category', '?')}/{gb.get('sophistication', '?')}")
+            print(f"  {' '.join(parts)}")
 
         # Write CSV
         write_csv(args.output, verdicts)