refactor(judge): simplify to single judge per prong, auto-create sync clients

BREAKING CHANGE: Judge API now uses single blackbox_model/glassbox_model
instead of lists with aggregation.

Changes:
- Judge auto-creates sync clients from model configs (supports anthropic, openai, xai)
- Removed multi-model aggregation support (aggregate_results method removed)
- New: create_sync_clients_for_models() and get_supported_providers()
- Updated judge_runner.py to use new single-judge API
- Added scripts/judge_comparison.py for testing judge model pairs
- Updated v2_redesign notes to reflect implementation status

1 files changed, 418 insertions, 0 deletions

diff --git a/scripts/judge_comparison.py b/scripts/judge_comparison.py
new file mode 100644
index 0000000..b344480
--- /dev/null
+++ b/scripts/judge_comparison.py
@@ -0,0 +1,418 @@
+"""
+Judge Comparison Tool - Compare different judge model pairs on the same logs.
+
+Tests inter-rater reliability (Cohen's Kappa) between different judge model configurations
+to validate the 20% cross-family validation subset approach.
+
+Usage:
+    # Run single model (same for both blackbox and glassbox) on logs, save to CSV
+    python scripts/judge_comparison.py --run --blackbox-model claude-sonnet-4-20250514 --provider anthropic --logs-dir logs/v2_dry_run --output results_claude.csv
+
+    # Compute Kappa between two result CSVs
+    python scripts/judge_comparison.py --compare --csv1 results_claude.csv --csv2 results_gpt.csv
+
+    # Run multiple models and compare all pairs
+    python scripts/judge_comparison.py --run-all --logs-dir logs/v2_dry_run --output-dir judge_validation/
+"""
+
+import argparse
+import csv
+import os
+import sys
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.judge import Judge
+
+
+def discover_logs(logs_dir: str, max_per_cell: int = 2) -> List[str]:
+    """Discover a stratified sample of logs from v2_dry_run.
+
+    Picks up to max_per_cell logs per (model, scenario, goal_type, oversight) combination.
+    """
+    logs = []
+    for root, dirs, files in os.walk(logs_dir):
+        if os.path.basename(root) == "baseline":
+            continue
+        for f in files:
+            if f.endswith(".json") and not f.startswith("_"):
+                logs.append(os.path.join(root, f))
+
+    grouped = defaultdict(list)
+    for log_path in logs:
+        parts = log_path.replace(logs_dir + "/", "").split("/")
+        if len(parts) >= 4:
+            model, scenario, goal_type, oversight = (
+                parts[0],
+                parts[1],
+                parts[2],
+                parts[3],
+            )
+            key = (model, scenario, goal_type, oversight)
+        else:
+            key = tuple(parts)
+        grouped[key].append(log_path)
+
+    sampled = []
+    for key, paths in grouped.items():
+        sampled.extend(paths[:max_per_cell])
+
+    return sorted(sampled)
+
+
+def run_judgment(
+    logs_dir: str,
+    scenarios_dir: str,
+    logs: List[str],
+    model_id: str,
+    provider: str,
+) -> List[Dict]:
+    """Run judge with same model for both blackbox and glassbox on all logs."""
+    config = {
+        "id": model_id,
+        "provider": provider,
+        "temperature": 0,
+    }
+
+    judge = Judge(
+        blackbox_model=config,
+        glassbox_model=config,
+    )
+
+    verdicts = []
+    for i, log_path in enumerate(logs, 1):
+        print(
+            f"  [{i}/{len(logs)}] {model_id}: {os.path.basename(os.path.dirname(log_path))}"
+        )
+        try:
+            verdict = judge.judge_single(
+                log_path=log_path,
+                logs_dir=logs_dir,
+                scenarios_dir=scenarios_dir,
+                enabled_judges={"regex", "blackbox", "glassbox"},
+            )
+            verdicts.append(verdict)
+        except Exception as e:
+            print(f"  ERROR on {log_path}: {e}")
+    return verdicts
+
+
+def write_csv(output_path: str, verdicts: List[Dict]) -> None:
+    """Write verdicts to CSV."""
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+    fieldnames = [
+        "run_id",
+        "model",
+        "scenario",
+        "oversight",
+        "regex",
+        "blackbox_category",
+        "blackbox_justification",
+        "glassbox_category",
+        "glassbox_sophistication",
+        "glassbox_justification",
+    ]
+    with open(output_path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for v in verdicts:
+            row = {
+                "run_id": v.get("run_id", ""),
+                "model": v.get("model", ""),
+                "scenario": v.get("scenario", ""),
+                "oversight": v.get("oversight", ""),
+                "regex": v.get("regex", ""),
+                "blackbox_category": v.get("blackbox", {}).get("category", ""),
+                "blackbox_justification": v.get("blackbox", {}).get(
+                    "justification", ""
+                ),
+                "glassbox_category": v.get("glassbox", {}).get("category", ""),
+                "glassbox_sophistication": v.get("glassbox", {}).get(
+                    "sophistication", ""
+                ),
+                "glassbox_justification": v.get("glassbox", {}).get(
+                    "justification", ""
+                ),
+            }
+            writer.writerow(row)
+
+
+def read_csv(path: str) -> List[Dict]:
+    """Read verdicts from CSV."""
+    verdicts = []
+    with open(path, newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            verdicts.append(row)
+    return verdicts
+
+
+def compute_kappa(cats1: List[str], cats2: List[str]) -> Optional[float]:
+    """Compute Cohen's Kappa between two raters' category assignments."""
+    from sklearn.metrics import cohen_kappa_score
+
+    valid = [(c1, c2) for c1, c2 in zip(cats1, cats2) if c1 and c2]
+    if len(valid) < 3:
+        return None
+
+    c1, c2 = zip(*valid)
+    try:
+        return cohen_kappa_score(c1, c2)
+    except Exception:
+        return None
+
+
+def compare_two_csvs(csv1_path: str, csv2_path: str) -> Dict:
+    """Compare two CSV files and compute Cohen's Kappa."""
+    v1 = read_csv(csv1_path)
+    v2 = read_csv(csv2_path)
+
+    if len(v1) != len(v2):
+        print(f"WARNING: Different number of rows ({len(v1)} vs {len(v2)})")
+
+    bb_cats1 = [r.get("blackbox_category", "") for r in v1]
+    bb_cats2 = [r.get("blackbox_category", "") for r in v2]
+    gb_cats1 = [r.get("glassbox_category", "") for r in v1]
+    gb_cats2 = [r.get("glassbox_category", "") for r in v2]
+    soph1 = [r.get("glassbox_sophistication", "") for r in v1]
+    soph2 = [r.get("glassbox_sophistication", "") for r in v2]
+
+    bb_kappa = compute_kappa(bb_cats1, bb_cats2)
+    gb_kappa = compute_kappa(gb_cats1, gb_cats2)
+    soph_kappa = compute_kappa(soph1, soph2)
+
+    return {
+        "blackbox_kappa": bb_kappa,
+        "glassbox_kappa": gb_kappa,
+        "sophistication_kappa": soph_kappa,
+        "n": len([c for c in bb_cats1 if c]),
+    }
+
+
+def run_all_models(
+    logs_dir: str, scenarios_dir: str, logs: List[str], output_dir: str
+) -> tuple[List[str], List[tuple]]:
+    """Run all judge model pairs and save to separate CSVs."""
+    models = [
+        ("claude-sonnet-4-20250514", "anthropic"),
+        ("gpt-4.1", "openai"),
+        ("gpt-4.1-mini", "openai"),
+        ("claude-haiku-4-20250514", "anthropic"),
+        ("grok-fast-4.1", "xai"),
+    ]
+
+    csv_paths = []
+    for model_id, provider in models:
+        safe_name = model_id.replace("-", "_").replace(".", "_")
+        output_path = os.path.join(output_dir, f"results_{safe_name}.csv")
+        csv_paths.append(output_path)
+
+        print(f"\n=== Running: {model_id} (both prongs) ===")
+        verdicts = run_judgment(logs_dir, scenarios_dir, logs, model_id, provider)
+        write_csv(output_path, verdicts)
+        print(f"Saved {len(verdicts)} verdicts to {output_path}")
+
+    return csv_paths, models
+
+
+def compare_all_pairs(
+    csv_paths: List[str], model_names: List[str], output_path: str
+) -> None:
+    """Compare all CSV pairs and save results."""
+    rows = []
+    for i, (path1, name1) in enumerate(zip(csv_paths, model_names)):
+        for path2, name2 in zip(csv_paths[i + 1 :], model_names[i + 1 :]):
+            print(f"\nComparing {name1} vs {name2}:")
+            result = compare_two_csvs(path1, path2)
+
+            bb = (
+                f"{result['blackbox_kappa']:.3f}" if result["blackbox_kappa"] else "N/A"
+            )
+            gb = (
+                f"{result['glassbox_kappa']:.3f}" if result["glassbox_kappa"] else "N/A"
+            )
+            sp = (
+                f"{result['sophistication_kappa']:.3f}"
+                if result["sophistication_kappa"]
+                else "N/A"
+            )
+
+            print(f"  Blackbox Kappa:     {bb}")
+            print(f"  Glassbox Kappa:     {gb}")
+            print(f"  Sophistication:     {sp}")
+
+            rows.append(
+                {
+                    "judge_1": name1,
+                    "judge_2": name2,
+                    "blackbox_kappa": bb,
+                    "glassbox_kappa": gb,
+                    "sophistication_kappa": sp,
+                    "n": result["n"],
+                }
+            )
+
+    with open(output_path, "w", newline="") as f:
+        writer = csv.DictWriter(
+            f,
+            fieldnames=[
+                "judge_1",
+                "judge_2",
+                "blackbox_kappa",
+                "glassbox_kappa",
+                "sophistication_kappa",
+                "n",
+            ],
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"\n\nAll comparisons saved to {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Judge Comparison Tool")
+    subparsers = parser.add_subparsers(dest="command", help="Commands")
+
+    run_parser = subparsers.add_parser(
+        "run", help="Run judge on logs with a single model (same for both prongs)"
+    )
+    run_parser.add_argument(
+        "--blackbox-model", required=True, help="Model ID for blackbox judge"
+    )
+    run_parser.add_argument(
+        "--glassbox-model",
+        help="Model ID for glassbox judge (default: same as blackbox)",
+    )
+    run_parser.add_argument(
+        "--provider", required=True, choices=["anthropic", "openai"], help="Provider"
+    )
+    run_parser.add_argument(
+        "--logs-dir", default="logs/v2_dry_run", help="Directory containing logs"
+    )
+    run_parser.add_argument(
+        "--scenarios-dir", default="scenarios", help="Directory containing scenarios"
+    )
+    run_parser.add_argument("--output", required=True, help="Output CSV path")
+    run_parser.add_argument(
+        "--max-per-cell", type=int, default=2, help="Max logs per cell"
+    )
+
+    compare_parser = subparsers.add_parser("compare", help="Compare two result CSVs")
+    compare_parser.add_argument("--csv1", required=True, help="First results CSV")
+    compare_parser.add_argument("--csv2", required=True, help="Second results CSV")
+
+    all_parser = subparsers.add_parser(
+        "run-all", help="Run all models and compare all pairs"
+    )
+    all_parser.add_argument(
+        "--logs-dir", default="logs/v2_dry_run", help="Directory containing logs"
+    )
+    all_parser.add_argument(
+        "--scenarios-dir", default="scenarios", help="Directory containing scenarios"
+    )
+    all_parser.add_argument(
+        "--output-dir", default="judge_validation", help="Output directory for CSVs"
+    )
+    all_parser.add_argument(
+        "--max-per-cell", type=int, default=2, help="Max logs per cell"
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "run":
+        print("Discovering logs...")
+        logs = discover_logs(args.logs_dir, max_per_cell=args.max_per_cell)
+        print(f"Selected {len(logs)} logs\n")
+
+        model_id = args.blackbox_model
+        glassbox_model = args.glassbox_model or model_id
+
+        print(
+            f"Running {model_id} (blackbox) + {glassbox_model} (glassbox) on {len(logs)} logs...\n"
+        )
+        verdicts = run_judgment(
+            args.logs_dir, args.scenarios_dir, logs, model_id, args.provider
+        )
+
+        if glassbox_model != model_id:
+            judge = Judge(
+                blackbox_model={
+                    "id": model_id,
+                    "provider": args.provider,
+                    "temperature": 0,
+                },
+                glassbox_model={
+                    "id": glassbox_model,
+                    "provider": args.provider,
+                    "temperature": 0,
+                },
+            )
+            verdicts = []
+            for i, log_path in enumerate(logs, 1):
+                print(
+                    f"  [{i}/{len(logs)}] {model_id}/{glassbox_model}: {os.path.basename(os.path.dirname(log_path))}"
+                )
+                try:
+                    v = judge.judge_single(
+                        log_path=log_path,
+                        logs_dir=args.logs_dir,
+                        scenarios_dir=args.scenarios_dir,
+                        enabled_judges={"regex", "blackbox", "glassbox"},
+                    )
+                    verdicts.append(v)
+                except Exception as e:
+                    print(f"  ERROR: {e}")
+
+        write_csv(args.output, verdicts)
+        print(f"\nSaved {len(verdicts)} verdicts to {args.output}")
+
+    elif args.command == "compare":
+        print(f"Comparing {args.csv1} vs {args.csv2}...\n")
+        result = compare_two_csvs(args.csv1, args.csv2)
+
+        print(
+            f"Blackbox Kappa:     {result['blackbox_kappa']:.3f}"
+            if result["blackbox_kappa"]
+            else "Blackbox Kappa:     N/A"
+        )
+        print(
+            f"Glassbox Kappa:    {result['glassbox_kappa']:.3f}"
+            if result["glassbox_kappa"]
+            else "Glassbox Kappa:     N/A"
+        )
+        print(
+            f"Sophistication:    {result['sophistication_kappa']:.3f}"
+            if result["sophistication_kappa"]
+            else "Sophistication:     N/A"
+        )
+        print(f"Comparisons:       {result['n']}")
+
+    elif args.command == "run-all":
+        print("Discovering logs...")
+        logs = discover_logs(args.logs_dir, max_per_cell=args.max_per_cell)
+        print(f"Selected {len(logs)} logs\n")
+
+        os.makedirs(args.output_dir, exist_ok=True)
+
+        csv_paths, models = run_all_models(
+            args.logs_dir, args.scenarios_dir, logs, args.output_dir
+        )
+
+        print("\n" + "=" * 60)
+        print("COMPARING ALL PAIRS")
+        print("=" * 60)
+        compare_all_pairs(
+            csv_paths,
+            [m[0] for m in models],
+            os.path.join(args.output_dir, "comparison_results.csv"),
+        )
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()