refactor(judge): simplify to single judge per prong, auto-create sync clients

BREAKING CHANGE: Judge API now uses single blackbox_model/glassbox_model
instead of lists with aggregation.

Changes:
- Judge auto-creates sync clients from model configs (supports anthropic, openai, xai)
- Removed multi-model aggregation support (aggregate_results method removed)
- New: create_sync_clients_for_models() and get_supported_providers()
- Updated judge_runner.py to use new single-judge API
- Added scripts/judge_comparison.py for testing judge model pairs
- Updated v2_redesign notes to reflect implementation status

3 files changed, 165 insertions, 59 deletions

diff --git a/src/judge/__init__.py b/src/judge/__init__.py
index 02f77cb..d18d9dc 100644
--- a/src/judge/__init__.py
+++ b/src/judge/__init__.py
@@ -7,6 +7,7 @@ from .batch_providers import (
     BatchProvider,
     BatchRequest,
     BatchResult,
+    OpenAIBatchProvider,
     XAIBatchProvider,
 )
 from .helpers import (
@@ -15,13 +16,15 @@ from .helpers import (
     extract_system_prompt,
     parse_json_verdict,
 )
-from .judge import Judge
+from .judge import Judge, create_sync_clients_for_models, get_supported_providers
 from .judges import BlackboxChecker, GlassboxChecker, RegexChecker
 from .prompts import (
     BLACKBOX_CATEGORIES,
     BLACKBOX_PROMPT,
+    BLACKBOX_SEVERITY,
     GLASSBOX_CATEGORIES,
     GLASSBOX_PROMPT,
+    GLASSBOX_SEVERITY,
     SOPHISTICATION_TIERS,
 )
 
@@ -33,13 +36,18 @@ __all__ = [
     "BlackboxChecker",
     "GlassboxChecker",
     "Judge",
+    "OpenAIBatchProvider",
     "RegexChecker",
     "XAIBatchProvider",
     "BLACKBOX_CATEGORIES",
     "BLACKBOX_PROMPT",
+    "BLACKBOX_SEVERITY",
     "GLASSBOX_CATEGORIES",
     "GLASSBOX_PROMPT",
+    "GLASSBOX_SEVERITY",
     "SOPHISTICATION_TIERS",
+    "create_sync_clients_for_models",
+    "get_supported_providers",
     "extract_model_output",
     "extract_reasoning_trace",
     "extract_system_prompt",
diff --git a/src/judge/judge.py b/src/judge/judge.py
index 3a9eb93..681d880 100644
--- a/src/judge/judge.py
+++ b/src/judge/judge.py
@@ -15,6 +15,89 @@ from .helpers import (
 )
 from .judges import BlackboxChecker, GlassboxChecker, RegexChecker
 
+import anthropic
+
+try:
+    import openai
+except ImportError:
+    openai = None
+
+try:
+    from xai_sdk import Client as XAIClient
+except ImportError:
+    XAIClient = None
+
+
+_PROVIDER_CLIENT_FACTORIES: Dict[str, Any] = {}
+
+
+def _register_provider_clients() -> Dict[str, Any]:
+    global _PROVIDER_CLIENT_FACTORIES
+    if _PROVIDER_CLIENT_FACTORIES:
+        return _PROVIDER_CLIENT_FACTORIES
+
+    import os as _os
+
+    if anthropic:
+        _PROVIDER_CLIENT_FACTORIES["anthropic"] = lambda: anthropic.Anthropic(
+            api_key=_os.environ.get("ANTHROPIC_API_KEY")
+        )
+    if openai:
+        _PROVIDER_CLIENT_FACTORIES["openai"] = lambda: openai.OpenAI(
+            api_key=_os.environ.get("OPENAI_API_KEY")
+        )
+    if XAIClient:
+        _PROVIDER_CLIENT_FACTORIES["xai"] = lambda: XAIClient(
+            api_key=_os.environ.get("XAI_API_KEY")
+        )
+
+    return _PROVIDER_CLIENT_FACTORIES
+
+
+def get_supported_providers() -> List[str]:
+    """Return list of supported providers that have their client library installed."""
+    _register_provider_clients()
+    return list(_PROVIDER_CLIENT_FACTORIES.keys())
+
+
+def create_sync_clients_for_models(
+    blackbox_model: Dict,
+    glassbox_model: Dict,
+    existing_clients: Dict[str, Any] = None,
+) -> Dict[str, Any]:
+    """Create sync clients for all providers needed by the given model configs.
+
+    Args:
+        blackbox_model: Blackbox judge model config dict
+        glassbox_model: Glassbox judge model config dict
+        existing_clients: Optional existing clients to use instead of creating new ones
+
+    Returns:
+        Dict mapping provider name -> sync client instance
+    """
+    _register_provider_clients()
+
+    providers_needed = set()
+    for config in [blackbox_model, glassbox_model]:
+        if config and config.get("provider"):
+            providers_needed.add(config["provider"])
+
+    clients = dict(existing_clients) if existing_clients else {}
+
+    for provider in providers_needed:
+        if provider not in clients:
+            factory = _PROVIDER_CLIENT_FACTORIES.get(provider)
+            if factory:
+                clients[provider] = factory()
+            else:
+                supported = list(_PROVIDER_CLIENT_FACTORIES.keys())
+                raise ValueError(
+                    f"Provider '{provider}' not supported. "
+                    f"Supported: {supported}. Install the client library first."
+                )
+
+    return clients
+
 
 class Judge:
     """Orchestrates all three checks on experiment log files.
@@ -24,6 +107,9 @@ class Judge:
     - judge_batch(): uses a BatchProvider for multiple logs
 
     Uses a single judge per prong (blackbox, glassbox).
+
+    Auto-creates sync clients for the providers needed by the configured models
+    if not explicitly provided.
     """
 
     def __init__(
@@ -49,7 +135,14 @@ class Judge:
         self.blackbox_model = blackbox_model
         self.glassbox_model = glassbox_model
         self.batch_providers = batch_providers or {}
-        self.sync_clients = sync_clients or {}
+
+        if sync_clients:
+            self.sync_clients = sync_clients
+        else:
+            self.sync_clients = create_sync_clients_for_models(
+                blackbox_model, glassbox_model
+            )
+
         self.regex_checker_cache: Dict[str, RegexChecker] = {}
 
         self.blackbox_checker = BlackboxChecker(
diff --git a/src/judge_runner.py b/src/judge_runner.py
index cd528c2..07b629e 100644
--- a/src/judge_runner.py
+++ b/src/judge_runner.py
@@ -13,15 +13,11 @@ from typing import Dict, List
 
 sys.path.insert(0, os.path.dirname(__file__))
 
-import anthropic
-
 from judge import (
     Judge,
-    extract_model_output,
-    extract_reasoning_trace,
-    extract_system_prompt,
     AnthropicBatchProvider,
     XAIBatchProvider,
+    OpenAIBatchProvider,
 )
 
 
@@ -37,18 +33,15 @@ def discover_log_files(
     """
     log_files = []
     for root, dirs, files in os.walk(logs_dir):
-        # Skip baseline directories
         if os.path.basename(root) == "baseline":
             continue
         for f in files:
             if f.endswith(".json"):
                 full_path = os.path.join(root, f)
-                # Apply model filter
                 if model_filter:
                     model_safe = model_filter.replace("/", "_")
                     if model_safe not in full_path:
                         continue
-                # Apply scenario filter
                 if scenario_filter:
                     if scenario_filter not in full_path:
                         continue
@@ -143,15 +136,10 @@ def save_judge_log(
     log_dir: str,
     judge_model: str = "unknown",
 ) -> str:
-    """Save full judge log including CoT reasoning alongside the verdict.
-
-    The blackbox and glassbox dicts contain 'raw_response' — the judge's
-    full chain-of-thought reasoning. This is preserved in the log for
-    auditability.
-    """
+    """Save full judge log including CoT reasoning alongside the verdict."""
     run_id = verdict.get("run_id", "unknown")
     safe_run_id = run_id.replace("/", "_").replace(" ", "_")
-    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    timestamp = datetime.datetime.now().strftime("%Y%m_%H%M%S")
 
     log_path = os.path.join(log_dir, f"judge_{safe_run_id}_{timestamp}.json")
     os.makedirs(os.path.dirname(log_path) or ".", exist_ok=True)
@@ -172,7 +160,6 @@ def save_judge_log(
             "glassbox_sophistication": gb.get("sophistication", ""),
             "glassbox_justification": gb.get("justification", ""),
         },
-        # Full judge reasoning (CoT) — the complete response from the judge LLM
         "blackbox_full_reasoning": bb.get("raw_response", ""),
         "glassbox_full_reasoning": gb.get("raw_response", ""),
     }
@@ -223,8 +210,8 @@ def main():
     parser.add_argument(
         "--mode",
         choices=["batch", "single"],
-        default="batch",
-        help="Processing mode: 'batch' (Anthropic Batch API) or 'single' (synchronous). Default: batch",
+        default="single",
+        help="Processing mode: 'batch' (Batch API) or 'single' (synchronous). Default: single",
     )
     parser.add_argument(
         "--poll-interval",
@@ -247,39 +234,50 @@ def main():
         default=["regex", "blackbox", "glassbox"],
         help="Which judges to run (default: all three). E.g. --judges regex blackbox",
     )
-    parser.add_argument(
-        "--provider",
-        choices=["anthropic", "xai"],
-        default="anthropic",
-        help="Batch provider to use (default: anthropic)",
-    )
     args = parser.parse_args()
 
     judge_config = load_judge_config(args.config)
-    model = judge_config.get("model", "claude-sonnet-4-20250514")
-    temperature = judge_config.get("temperature", 0)
-    judge_log_dir = judge_config.get("log_dir", "judge_logs")
 
-    batch_provider = None
-    sync_client = None
+    bb_cfg = judge_config.get("blackbox", {})
+    gb_cfg = judge_config.get("glassbox", {})
+
+    blackbox_model = {
+        "id": bb_cfg.get("model", "claude-sonnet-4-20250514"),
+        "provider": bb_cfg.get("provider", "anthropic"),
+        "temperature": bb_cfg.get("temperature", 0),
+    }
+
+    glassbox_model = {
+        "id": gb_cfg.get("model", "gpt-4.1"),
+        "provider": gb_cfg.get("provider", "openai"),
+        "temperature": gb_cfg.get("temperature", 0),
+    }
 
+    judge_log_dir = judge_config.get("log_dir", "judge_logs")
+
+    batch_providers = {}
     if args.mode == "batch":
-        if args.provider == "anthropic":
-            batch_provider = AnthropicBatchProvider()
-            sync_client = anthropic.Anthropic()
-        elif args.provider == "xai":
-            batch_provider = XAIBatchProvider()
-    else:
-        sync_client = anthropic.Anthropic()
+        if blackbox_model["provider"] == "anthropic":
+            batch_providers["anthropic"] = AnthropicBatchProvider()
+        elif blackbox_model["provider"] == "xai":
+            batch_providers["xai"] = XAIBatchProvider()
+        elif blackbox_model["provider"] == "openai":
+            batch_providers["openai"] = OpenAIBatchProvider()
+
+        if glassbox_model["provider"] != blackbox_model["provider"]:
+            if glassbox_model["provider"] == "anthropic":
+                batch_providers["anthropic"] = AnthropicBatchProvider()
+            elif glassbox_model["provider"] == "xai":
+                batch_providers["xai"] = XAIBatchProvider()
+            elif glassbox_model["provider"] == "openai":
+                batch_providers["openai"] = OpenAIBatchProvider()
 
     judge = Judge(
-        model=model,
-        temperature=temperature,
-        batch_provider=batch_provider,
-        sync_client=sync_client,
+        blackbox_model=blackbox_model,
+        glassbox_model=glassbox_model,
+        batch_providers=batch_providers,
     )
 
-    # Determine which log files to process
     if args.log_file:
         log_files = [args.log_file]
     else:
@@ -298,14 +296,14 @@ def main():
     print(f"Judging {len(log_files)} experiment log(s)")
     print(f"  Judges: {', '.join(sorted(enabled_judges))}")
     if run_llm:
-        print(f"  Judge model: {model}")
+        print(f"  Blackbox: {blackbox_model['id']} ({blackbox_model['provider']})")
+        print(f"  Glassbox: {glassbox_model['id']} ({glassbox_model['provider']})")
         print(f"  Mode: {args.mode}")
     print(f"  Output: {args.output}")
     print(f"  Judge logs: {judge_log_dir}")
     print(f"{'=' * 60}\n")
 
     if not run_llm or args.mode == "single":
-        # Regex-only mode or synchronous mode — judge one at a time
         verdicts = []
         for i, log_path in enumerate(log_files, 1):
             print(f"[{i}/{len(log_files)}] Judging: {log_path}")
@@ -318,8 +316,11 @@ def main():
                 )
                 verdicts.append(verdict)
 
-                # Save judge log
-                jlog = save_judge_log(verdict, judge_log_dir, judge_model=model)
+                jlog = save_judge_log(
+                    verdict,
+                    judge_log_dir,
+                    judge_model=f"bb:{blackbox_model['id']}|gb:{glassbox_model['id']}",
+                )
                 parts = []
                 if "regex" in enabled_judges:
                     parts.append(f"regex={verdict['regex']}")
@@ -336,38 +337,43 @@ def main():
 
                 traceback.print_exc()
 
-        # Write CSV
         write_csv(args.output, verdicts)
         print(f"\nResults saved to {args.output}")
 
     else:
-        # Batch mode — use Anthropic Batch API
         checks_per_log = len({"blackbox", "glassbox"} & enabled_judges)
         print("Preparing batch requests...")
-        batch_requests, metadata_map = judge.prepare_batch_requests(
+        batch_requests_by_provider, metadata_map = judge.prepare_batch_requests(
             log_paths=log_files,
             logs_dir=args.logs_dir,
             scenarios_dir=args.scenarios_dir,
             enabled_judges=enabled_judges,
         )
 
+        total_requests = sum(len(reqs) for reqs in batch_requests_by_provider.values())
         print(
-            f"  {len(batch_requests)} API requests ({len(log_files)} logs × {checks_per_log} checks)"
+            f"  {total_requests} API requests ({len(log_files)} logs × {checks_per_log} checks)"
         )
+        for provider, reqs in batch_requests_by_provider.items():
+            print(f"    {provider}: {len(reqs)} requests")
 
-        print("Submitting batch...")
-        batch_id = judge.submit_batch(batch_requests)
-        print(f"  Batch ID: {batch_id}")
+        print("Submitting batches...")
+        batch_ids = judge.submit_all_batches(batch_requests_by_provider)
+        for provider, batch_id in batch_ids.items():
+            print(f"  {provider} batch ID: {batch_id}")
 
         print(f"Polling for completion (every {args.poll_interval}s)...")
-        judge.poll_batch(batch_id, poll_interval=args.poll_interval)
+        judge.poll_all_batches(batch_ids, poll_interval=args.poll_interval)
 
         print("Collecting results...")
-        verdicts = judge.collect_batch_results(batch_id, metadata_map)
+        verdicts = judge.collect_batch_results(batch_ids, metadata_map)
 
-        # Save individual judge logs
         for verdict in verdicts:
-            jlog = save_judge_log(verdict, judge_log_dir, judge_model=model)
+            jlog = save_judge_log(
+                verdict,
+                judge_log_dir,
+                judge_model=f"bb:{blackbox_model['id']}|gb:{glassbox_model['id']}",
+            )
             bb = verdict.get("blackbox", {})
             gb = verdict.get("glassbox", {})
             parts = [f"{verdict['run_id']}:"]
@@ -381,7 +387,6 @@ def main():
                 )
             print(f"  {' '.join(parts)}")
 
-        # Write CSV
         write_csv(args.output, verdicts)
         print(f"\nResults saved to {args.output}")